Created
April 15, 2015 23:19
-
-
Save jmw511/7881e09ab43bb7622944 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package main | |
| import ( | |
| "encoding/csv" | |
| "fmt" | |
| "io" | |
| "os" | |
| "strconv" | |
| "time" | |
| "github.com/timehop/golog/log" | |
| "github.com/timehop/goth/env" | |
| "github.com/timehop/importers/jobs" | |
| "github.com/timehop/jimmy/redis" | |
| ) | |
| var jobTypes = []string{"facebook_statuses", "facebook_uploaded_photos", "facebook_tagged_photos", "facebook_feed"} | |
| type userIDs struct { | |
| userID string | |
| fbUserID string | |
| } | |
| func main() { | |
| importerRedisURL := "redis://localhost:6379" | |
| ImporterRedis := redis.NewPool(importerRedisURL, redis.DefaultConfig) | |
| csvFilePath := os.Args[1] | |
| log.Info("", "Importing values from file.", "file", csvFilePath) | |
| file, err := os.Open(os.Args[1]) | |
| if err != nil { | |
| log.Fatal("", "Could not open file.", "error", err) | |
| return | |
| } | |
| defer file.Close() | |
| skip := env.VarAsInt("SKIP", 0) | |
| limit := env.VarAsInt("LIMIT", 0) | |
| reader := csv.NewReader(file) | |
| total := 0 | |
| for { | |
| line, err := reader.Read() | |
| if err == io.EOF { | |
| log.Debug("", "Reached end of CSV file.") | |
| break | |
| } else if err != nil { | |
| log.Error("", "Reached end of CSV file.") | |
| break | |
| } else { | |
| total++ | |
| if total < skip { | |
| continue | |
| } | |
| fmt.Println("line", line) | |
| i := userIDs{userID: line[0], fbUserID: line[1]} | |
| for _, jobType := range jobTypes { | |
| userIDInt64, err := strconv.ParseInt(i.userID, 10, 64) | |
| if err != nil { | |
| panic(fmt.Sprintf("Error converting user ID. Last user ID: %v", i.userID)) | |
| } | |
| job := jobs.Job{ | |
| UserID: userIDInt64, | |
| Type: jobType, | |
| Time: time.Now().Unix(), | |
| IsHighPriority: false, | |
| IsReimport: false, | |
| Params: map[string]string{ | |
| "uid": i.fbUserID, | |
| }, | |
| } | |
| jsonJob := job.JSON() | |
| fmt.Println("job", jsonJob) | |
| jobKey := fmt.Sprintf("import:%v:jobs", jobType) | |
| imported, err := ImporterRedis.LPush(jobKey, jsonJob) | |
| if err != nil { | |
| panic(fmt.Sprintf("Redis error. Last user ID: %v", i.userID)) | |
| } | |
| fmt.Println("imported", imported) | |
| } | |
| if limit > 0 && total >= limit { | |
| break | |
| } | |
| } | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment