Last active
December 16, 2015 14:59
-
-
Save woodywang/5452928 to your computer and use it in GitHub Desktop.
对以下说法的质疑: $in 操作一次是快,但是在 list 量大的时候会造成全局的阻塞,造成大量的慢查询(查询查日志中都有记录)。因为目前一个库要对应的查询各种各样,每秒的读操作要上千或者几千,并且 mongodb 是单进程在跑,所以我们要遵循的原则就是单次请求要尽量快的完成。$in 操作,尽量少用,即使要用,list 中的元素也尽量的少。
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package com.woodywang; | |
import com.mongodb.*; | |
public class Benchmark { | |
/** | |
* 使用线程模拟多个客户端并发请求 | |
*/ | |
static class Finder implements Runnable { | |
private int size; | |
private int loop; | |
private String id; | |
private DBCollection collection; | |
/** | |
* 构造执行 find 操作的客户端 | |
* | |
* @param size $in 条件长度 | |
* @param loop 重复执行次数(使统计结果更明显) | |
* @param id 线程标识 | |
* @throws Exception | |
*/ | |
public Finder(int size, int loop, String id) throws Exception { | |
this.size = size; | |
this.loop = loop; | |
this.id = id; | |
MongoClient client = new MongoClient("127.0.0.1", 27017); | |
DB db = client.getDB("test"); | |
this.collection = db.getCollection("test"); | |
} | |
private String[] generateIds() { | |
String[] ids = new String[this.size]; | |
for (int i = 0; i < this.size; i++) { | |
ids[i] = String.format("%08d", (int) (Math.random() * 1000000)); | |
} | |
return ids; | |
} | |
public void run() { | |
long totalTime = 0; | |
for (int i = 0; i < this.loop; i++) { | |
String[] ids = generateIds(); | |
BasicDBList idList = new BasicDBList(); | |
for (String id : ids) { | |
idList.add(id); | |
} | |
DBObject condition = new BasicDBObject(); | |
condition.put("uid", new BasicDBObject("$in", idList)); | |
long startTime = System.currentTimeMillis(); | |
DBCursor cursor = collection.find(condition); | |
cursor.count(); | |
long endTime = System.currentTimeMillis(); | |
totalTime += endTime - startTime; | |
} | |
System.out.println(id + ": DataSize: " + (this.loop * this.size) + "; Time: " + totalTime); | |
} | |
} | |
public static void main(String[] args) throws Exception { | |
// 创建一个带有大量 ID 的 $in 条件查询测试,如执行 $in 条件有 2000 个ID,执行一次 | |
Thread tLong = new Thread(new Finder(2000, 1, "LongThread")); | |
// 创建若干个小数据量的查询测试 | |
int countShortThreads = 100; | |
Thread[] threads = new Thread[countShortThreads + 1]; | |
for (int i = 0; i < countShortThreads; i++) { | |
// 每个小数据量的测试定义为:每次查询只使用 1 个 ID,反复执行 2000 次 | |
threads[i] = new Thread(new Finder(1, 2000, "ShortThread-" + i)); | |
} | |
threads[countShortThreads] = tLong; | |
for (Thread t : threads) { | |
t.start(); | |
} | |
for (Thread t : threads) { | |
t.join(); | |
} | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package com.woodywang; | |
import com.mongodb.BasicDBObject; | |
import com.mongodb.DB; | |
import com.mongodb.DBCollection; | |
import com.mongodb.MongoClient; | |
/** | |
* Created with IntelliJ IDEA. | |
* User: woody | |
* Date: 4/24/13 | |
* Time: 8:16 PM | |
* To change this template use File | Settings | File Templates. | |
*/ | |
public class Generator { | |
private static void fillData(DBCollection collection, int size) throws Exception { | |
for (int i = 0; i < size; i++) { | |
String uid = String.format("%08d", i); | |
BasicDBObject doc = new BasicDBObject("uid", uid); | |
doc.append("text", generateString(15)); | |
collection.insert(doc); | |
} | |
} | |
private static String generateString(int size) { | |
String str = "abcdefghijklmnopqrstuvwxyz"; | |
StringBuilder builder = new StringBuilder(); | |
for (int i = 0; i < size; i++) { | |
int index = (int) (Math.random() * str.length()); | |
char c = str.charAt(index); | |
builder.append(c); | |
} | |
return builder.toString(); | |
} | |
public static void main(String[] args) throws Exception { | |
MongoClient client = new MongoClient("localhost"); | |
DB db = client.getDB("test"); | |
DBCollection collection = db.getCollection("test"); | |
fillData(collection, 1000000); | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" | |
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> | |
<modelVersion>4.0.0</modelVersion> | |
<groupId>com.woodywang</groupId> | |
<artifactId>mongodb-benchmark</artifactId> | |
<packaging>jar</packaging> | |
<version>1.0-SNAPSHOT</version> | |
<name>mongodb-benchmark</name> | |
<url>http://maven.apache.org</url> | |
<dependencies> | |
<dependency> | |
<groupId>junit</groupId> | |
<artifactId>junit</artifactId> | |
<version>3.8.1</version> | |
<scope>test</scope> | |
</dependency> | |
<dependency> | |
<groupId>org.mongodb</groupId> | |
<artifactId>mongo-java-driver</artifactId> | |
<version>2.11.1</version> | |
</dependency> | |
</dependencies> | |
</project> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment