Last active
July 15, 2023 11:26
-
-
Save puzpuzpuz/23e875b793200cf1c8b6cc19f85719f9 to your computer and use it in GitHub Desktop.
Reproducer for heavy O3 ingestion
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package main | |
| import ( | |
| "context" | |
| "fmt" | |
| "log" | |
| "math/rand" | |
| "strconv" | |
| "strings" | |
| "time" | |
| qdb "github.com/questdb/go-questdb-client" | |
| ) | |
| const ( | |
| tableCount = 24 | |
| symbolCount = 5 | |
| distinctSymbols = 100 | |
| strCount = 9 | |
| doubleCount = 90 | |
| longCount = 8 | |
| batchSize = 10_000 | |
| ) | |
| var ( | |
| strValue = strings.Repeat("a", 64) | |
| ) | |
| func main() { | |
| ctx := context.TODO() | |
| sender, err := qdb.NewLineSender(ctx) | |
| if err != nil { | |
| log.Fatal(err) | |
| } | |
| defer sender.Close() | |
| rn := 0 | |
| t := time.Now().UnixNano() | |
| s := time.Now() | |
| for { | |
| for tn := 0; tn < tableCount; tn++ { | |
| sender.Table("heavy_03_repro_table_" + strconv.Itoa(tn)) | |
| for cn := 0; cn < symbolCount; cn++ { | |
| sender.Symbol("sym_col_"+strconv.Itoa(cn), "what_a_sym_value_"+strconv.Itoa(rn%distinctSymbols)) | |
| } | |
| for cn := 0; cn < strCount; cn++ { | |
| sender.StringColumn("string_col_"+strconv.Itoa(cn), strValue) | |
| } | |
| for cn := 0; cn < doubleCount; cn++ { | |
| sender.Float64Column("double_col_"+strconv.Itoa(cn), float64(rn)) | |
| } | |
| for cn := 0; cn < longCount; cn++ { | |
| sender.Int64Column("long_col_"+strconv.Itoa(cn), int64(rn)) | |
| } | |
| lt := t | |
| // Occasionally we jump back in time | |
| if rand.Int63n(100) > 95 { | |
| lt -= time.Hour.Nanoseconds() | |
| } else { | |
| lt += (rand.Int63n(10) - 5) * time.Minute.Nanoseconds() | |
| } | |
| err = sender.At(ctx, lt) | |
| if err != nil { | |
| log.Fatal(err) | |
| } | |
| rn++ | |
| if rn%batchSize == 0 { | |
| err = sender.Flush(ctx) | |
| if err != nil { | |
| log.Fatal(err) | |
| } | |
| fmt.Printf("Throughput: %d rows/s\n", int(batchSize/time.Since(s).Seconds())) | |
| s = time.Now() | |
| } | |
| } | |
| // ~14.4M (4*3.6) rows per hour | |
| t += time.Millisecond.Nanoseconds() / 4 | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment