This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
response = s3client.put_object( | |
Body=gz_body.getvalue(), | |
Bucket=bucket_name, | |
Key=s3_path, | |
ContentType='text/plain', # the original type | |
ContentEncoding='gzip', # MUST have or browsers will error | |
ACL='public-read' | |
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"query": { | |
"bool": { | |
"filter": [ | |
{ | |
"range": { | |
"@timestamp": { | |
"from": "{{period_end}}||-1m", | |
"to": "{{period_end}}", | |
"include_lower": true, |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"query": { | |
"bool": { | |
"filter": [ | |
{ | |
"range": { | |
"@timestamp": { | |
"from": "{{period_end}}||-1m", | |
"to": "{{period_end}}", | |
"include_lower": true, |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"api": { | |
"invokeUrl": "https://5bltcq602h.execute-api.us-west-2.amazonaws.com/prod" | |
}, | |
"cognito" : { | |
"REGION":"us-east-1", | |
"USER_POOL_ID":"us-east-1_************", | |
"APP_CLIENT_ID" :"34uglnub******md0d1d" | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pandas as pd | |
import numpy as np | |
import matplotlib.pyplot as plt | |
import seaborn as sns | |
'exec(%matplotlib inline)' | |
from urllib.request import urlopen | |
from bs4 import BeautifulSoup | |
import time | |
from selenium import webdriver | |
import MySQLdb |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
sql = "INSERT INTO `newsarticles` (`source`, `sid`, `time`, `title`, `body`) VALUES (%s, %s, %s, %s, %s);" # those names depends on your database. | |
try: | |
cur.execute(sql, ('adaderana',0, date, title, contents)) | |
db.commit() | |
except (MySQLdb.Error, MySQLdb.Warning) as e: | |
print ("SQL Error") | |
raise e |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
with open ('news.csv','w',encoding="utf-8") as file: # you have to create .csv file call news.csv before use this | |
writer=csv.writer(file) | |
writer.writerow(course) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
with open("news.txt", "w", encoding="utf-8") as file: | |
file.write(date) | |
file.write(title.decode(encoding='UTF-8')) | |
file.write(contents.decode(encoding='UTF-8')) | |
file.close() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
for link in all_div: | |
news_container = link.find_all("div",{"class" : "story-text"}) | |
for news in news_container: | |
h2_tags = news.find_all("h2") | |
for url in h2_tags: | |
a_tags = url.find_all('a') | |
for end_point in a_tags: | |
#print(end_point.get("href")) | |
url_ind = "http://sinhala.adaderana.lk/"+end_point.get("href") | |
driver_ind = webdriver.Firefox(executable_path=r'D:\apps\anaconda\geckodriver.exe') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from bs4 import BeautifulSoup | |
import time | |
from selenium import webdriver | |
import MySQLdb | |
from datetime import datetime | |
import csv | |
url = "http://sinhala.adaderana.lk/sinhala-hot-news.php" #this is website url | |
db = MySQLdb.connect(host="localhost", |