Last active
August 5, 2021 06:34
-
-
Save sowich/c334c33777239c2b5e28632caf0e3fe5 to your computer and use it in GitHub Desktop.
parser.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"fmt" | |
"log" | |
"net/http" | |
"strings" | |
"sync" | |
"time" | |
"github.com/PuerkitoBio/goquery" | |
) | |
func getTitle(url string) (title string, timeDiff int64) { | |
t := time.Now() | |
res, err := http.Get(url) | |
if err != nil { | |
fmt.Println(err) | |
} | |
defer res.Body.Close() | |
doc, err := goquery.NewDocumentFromReader(res.Body) | |
if err != nil { | |
log.Fatalln(err) | |
} | |
return doc.Find("title").Text(), time.Since(t).Milliseconds() | |
} | |
func main() { | |
res, err := http.Get("https://tproger.ru") | |
if err != nil { | |
log.Fatalln(err) | |
} | |
defer res.Body.Close() | |
doc, err := goquery.NewDocumentFromReader(res.Body) | |
if err != nil { | |
log.Fatal(err) | |
} | |
var urls []string | |
doc.Find("a").Each(func(i int, s *goquery.Selection) { | |
href, _ := s.Attr("href") | |
if strings.Contains(href, "tag") { | |
urls = append(urls, "https://tproger.ru"+href) | |
} | |
}) | |
var wg sync.WaitGroup | |
for _, url := range urls { | |
wg.Add(1) | |
go func(url string) { | |
title, timeDiff := getTitle(url) | |
fmt.Printf("Title: %s, Time: %dms\n", title, timeDiff) | |
wg.Done() | |
}(url) | |
} | |
wg.Wait() | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment