Last active
October 15, 2023 21:01
-
-
Save oprietop/e99c2b8159e7ec3b5cfd5c177fd95301 to your computer and use it in GitHub Desktop.
golang snippet template using structs, goroutines with limited paralellism, http requests, json unmarshall, safe access to variables
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"os" | |
"fmt" | |
"time" | |
"io" | |
"io/ioutil" | |
"sync" | |
"log" | |
"net/http" | |
"net/http/httputil" | |
"net/url" | |
"strings" | |
"bytes" | |
"encoding/json" | |
"sort" | |
) | |
// Max number of running goroutines | |
const MAX_CONCURRENT_JOBS = 10 | |
// Main struct | |
type RunTasks struct { | |
mutex sync.Mutex | |
client *http.Client | |
waitGroup *sync.WaitGroup | |
waitChan chan struct{} | |
pages map[int][]byte | |
debug bool | |
} | |
// Struct generator | |
func NewRunTasks() *RunTasks { | |
client := &http.Client{ | |
Timeout: 10 * time.Second, | |
} | |
// Check if we launched the program with the debug arg | |
debug := false | |
if len(os.Args) == 2 { | |
if os.Args[1] == "debug" { | |
debug = true | |
} | |
} | |
return &RunTasks{ | |
client: client, | |
waitGroup: &sync.WaitGroup{}, | |
waitChan: make(chan struct{}, MAX_CONCURRENT_JOBS), | |
pages: map[int][]byte{}, | |
debug: debug, | |
} | |
} | |
// HTTP fetch | |
func (rt *RunTasks) fetch( | |
method string, | |
uri string, | |
form map[string]string, | |
data []byte, | |
headers map[string]string, | |
debug bool, | |
) (result []byte, err error) { | |
// Generate the request body | |
var rBody io.Reader = nil | |
// Create a request body with a FORM | |
if form != nil { | |
args := url.Values{} | |
for k, v := range form { | |
args.Set(k, v) | |
} | |
rBody = strings.NewReader(args.Encode()) | |
} | |
// Create a request body with RAW data | |
if data != nil { | |
rBody = bytes.NewBuffer(data) | |
} | |
// Create the request | |
req, err := http.NewRequest(method, uri, rBody) | |
if err != nil { | |
return nil, err | |
} | |
// Add headers | |
for k, v := range headers { | |
req.Header.Add(k, v) | |
} | |
// Print request headers and body on bebug | |
if debug == true { | |
reqDump, err := httputil.DumpRequestOut(req, true) | |
if err != nil { | |
log.Fatal(err) | |
} | |
log.Printf(">> REQUEST:\n%s\n", string(reqDump)) | |
} | |
// Perform the request | |
res, err := rt.client.Do(req) | |
if err != nil { | |
return nil, err | |
} | |
defer res.Body.Close() | |
// Print response headers and body on bebug | |
if debug == true { | |
respDump, err := httputil.DumpResponse(res, true) | |
if err != nil { | |
log.Fatal(err) | |
} | |
log.Printf("<< RESPONSE:\n%s\n", string(respDump)) | |
} | |
// Retrieve the body | |
body, err := ioutil.ReadAll(res.Body) | |
if err != nil { | |
return nil, err | |
} | |
return body, nil | |
} | |
// Update shared variables avoiding race conditions | |
func (rt *RunTasks) updateMap(key int, value []byte) (err error) { | |
// Unlock on exit | |
defer rt.mutex.Unlock() | |
// While locked only one goroutine can access the map | |
rt.mutex.Lock() | |
// Now we can safely access the variable | |
rt.pages[key] = value | |
return nil | |
} | |
// Job to run as goroutine | |
func (rt *RunTasks) job(index int) (err error) { | |
// Decrement the counter when the goroutine completes | |
defer rt.waitGroup.Done() | |
url := fmt.Sprintf("https://pokeapi.co/api/v2/pokemon/%d", index) | |
// Fetch the page | |
pageBytes, err := rt.fetch("GET", url, nil, nil, nil, rt.debug) | |
if err != nil { | |
log.Printf("Error %s while fetching %s", err, url) | |
} | |
// Save the results in a map | |
rt.updateMap(index, pageBytes) | |
return nil | |
} | |
func main() { | |
// Create a runtasks struct | |
rt := NewRunTasks() | |
// Fetch all 151 pokemons from pokeapi | |
for count := 1; count <= 151; count++ { | |
// Put an empty struct in the fixed size channel | |
// This will block and wait when channel is full | |
rt.waitChan <- struct{}{} | |
// Increment the counter | |
rt.waitGroup.Add(1) | |
go func(count int) { | |
log.Printf(">> Job %v added. Got %v queued jobs.\n", count, len(rt.waitChan)) | |
rt.job(count) | |
// Revome an item from the channel | |
<- rt.waitChan | |
log.Printf("<< Job %v done. Got %v queued jobs.\n", count, len(rt.waitChan)) | |
}(count) | |
} | |
// Wait for the goroutines to finish. | |
rt.waitGroup.Wait() | |
// Create a slice with the keys of the pages map | |
keys := make([]int, 0, len(rt.pages)) | |
for key := range rt.pages { | |
keys = append(keys, key) | |
} | |
// Sort the list | |
sort.Ints(keys) | |
// Print our map | |
for index, k := range keys { | |
page := rt.pages[k] | |
// Unmarshall the Json | |
var content map[string]interface{} | |
json.Unmarshal(page, &content) | |
// Print info for each result | |
fmt.Println("Index:", index, "Length:", len(page), "Name:", content["name"], "Exp:", content["base_experience"]) | |
} | |
os.Exit(0) | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment