Skip to content

Instantly share code, notes, and snippets.

@jack980517

jack980517/maimainet_musicscore.py

Last active November 18, 2020 00:58
Show Gist options
  • Save jack980517/44a365955322830cb3ea3d14f2c67fd9 to your computer and use it in GitHub Desktop.
Save jack980517/44a365955322830cb3ea3d14f2c67fd9 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
maimainet_musicscore.py
#!python2
#coding:utf8
#Usage:
#Install Python 2.7
#Execute `pip install requests lxml` in Powershell/cmd/Terminal
#Supply your own Sega ID & password in `username_password.txt`
#Run the script
#If errors occur, just re-run the script
import requests,json,os
from lxml import html
def check_if_ok(result):
if not result.ok:
print result.url,result.status_code,result.reason
exit()
def post(session_requests,post_url,post_data,referer_url):
useragent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
headers={'user-agent':useragent,'referer':referer_url}
result=session_requests.post(post_url,data=post_data,headers=headers)
check_if_ok(result)
return result
def get(session_requests,get_url,referer_url):
useragent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
headers={'user-agent':useragent,'referer':referer_url}
result=session_requests.get(get_url,headers=headers)
check_if_ok(result)
return result
session_requests=requests.session()
#login
print 'login'
username,password=open('username_password.txt').read().split('\n')
login_page_url='https://maimai-net.com/maimai-mobile/'
login_post_url='https://maimai-net.com/maimai-mobile/index/submit/'
login_post_data={
'segaId':username,
'passWd':password,
'x':0,
'y':0
}
post(session_requests,login_post_url,login_post_data,login_page_url)
#musicscore
difficulties=['easy','basic','advanced','expert','master','remaster','utage']
baseurl='https://maimai-net.com/maimai-mobile/music/%sGenre/'
for i in difficulties:
print i
if os.path.exists('music_scores_%s.json'%i): continue
response=get(session_requests,baseurl%i,'https://maimai-net.com/maimai-mobile/music/')
response.encoding='utf8'
content=response.text
tree=html.fromstring(content)
data_diff=tree.xpath('/html/body/div[@class="data_%s"]'%i)[0]
accordion=data_diff.xpath('div[@id="accordion"]')[0]
last_played_date=[x.strip() for x in accordion.xpath('ul/li/div[1]/text()')]
play_count=[0 if x==u'---回' else int(x[:-1]) for x in accordion.xpath('ul/li/table/tr[1]/td[2]/text()')]
high_score=[0 if x==u'---' else int(x.replace(',','')) for x in accordion.xpath('ul/li/table/tr[2]/td[2]/text()')]
high_sync_rate=[0 if x==u'---%' else float(x[:-1]) for x in accordion.xpath('ul/li/table/tr[3]/td[2]/text()')]
music_id=accordion.xpath('ul/li/div[2]/form/input[@name="musicId"]/@value')
keys=['last_played_date','play_count','high_score','high_sync_rate','music_id']
if i=='utage':
music_diff=accordion.xpath('ul/li/div[2]/form/input[@name="musicDiff"]/@value')
keys.append('music_diff')
music_score_data=[{x:eval(x+'[y]') for x in keys} for y in range(len(music_id))]
h3=accordion.xpath('h3')
for j in range(len(h3)):
title=h3[j].text.strip()
achievement_rate=0
badges=[]
played_flag=title==''
if played_flag:
title=h3[j].xpath('div[1]/text()')[0]
achievement_rate=float(h3[j].xpath('div[2]/table/tr/td[1]/text()')[1].strip()[:-1])
badges=[x.split('icon_')[1].split('.png')[0] for x in h3[j].xpath('div[2]/table/tr/td[2]/img/@src')]
music_score_data[j].update({x:eval(x) for x in ['title','achievement_rate','badges']})
open('music_scores_%s.json'%i,'w').write(json.dumps(music_score_data,indent=4,ensure_ascii=False).encode('utf8'))
Raw
maimainet_musicscore_ranking.py
#!python2
#coding:utf8
#Usage:
#Install Python 2.7
#Execute `pip install requests lxml` in Powershell/cmd/Terminal
#Supply your own Sega ID & password in `username_password.txt`
#Run the script
#If errors occur, just re-run the script
import requests,json,os,glob,re
from lxml import html
def check_if_ok(result):
if not result.ok:
print result.url,result.status_code,result.reason
exit()
def post(session_requests,post_url,post_data,referer_url):
useragent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
headers={'user-agent':useragent,'referer':referer_url}
result=session_requests.post(post_url,data=post_data,headers=headers)
check_if_ok(result)
return result
def get(session_requests,get_url,referer_url):
useragent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
headers={'user-agent':useragent,'referer':referer_url}
result=session_requests.get(get_url,headers=headers)
check_if_ok(result)
return result
session_requests=requests.session()
#login
print 'login'
username,password=open('username_password.txt').read().split('\n')
login_page_url='https://maimai-net.com/maimai-mobile/'
login_post_url='https://maimai-net.com/maimai-mobile/index/submit/'
login_post_data={
'segaId':username,
'passWd':password,
'x':0,
'y':0
}
post(session_requests,login_post_url,login_post_data,login_page_url)
#musicscore_ranking
charts=['easy','basic','advanced','expert','master','remaster','utage']
baseurl='https://maimai-net.com/maimai-mobile/music/%sGenre/'
for i in charts:
print i
if not os.path.exists(i): os.mkdir(i)
musicscore_response=get(session_requests,baseurl%i,'https://maimai-net.com/maimai-mobile/music/')
musicscore_response.encoding='utf8'
musicscore_content=musicscore_response.text
tree=html.fromstring(musicscore_content)
data_diff=tree.xpath('/html/body/div[@class="data_%s"]'%i)[0]
accordion=data_diff.xpath('div[@id="accordion"]')[0]
keys=['difficulty','musicDiff','musicId']
data={x: accordion.xpath('ul/li/div[2]/form/input[@name="%s"]/@value'%x) for x in keys}
ranking_post_data_list=[{y: eval('data["%s"][x]'%y) for y in keys} for x in range(len(data['musicId']))]
for j in ranking_post_data_list:
filename_prefix=j['musicId']+'_'+j['musicDiff'] if i=='utage' else j['musicId']
print filename_prefix
if glob.glob(i+'/'+filename_prefix+'_*.json')!=[]: continue
post(session_requests,"https://maimai-net.com/maimai-mobile/music/type/",j,baseurl%i)
ranking_data={}
for k in ['friend/','']:
ranking_type='all' if k=='' else 'friend'
ranking_response=get(session_requests,'https://maimai-net.com/maimai-mobile/music/ranking/'+k,'https://maimai-net.com/maimai-mobile/music/rankingType')
ranking_response.encode='utf8'
ranking_content=ranking_response.text
tree=html.fromstring(ranking_content)
music_title=tree.xpath('/html/body/div[@class="status5"]/p[2]/text()')[0]
invalid_chars=r'\/:*?"<>|'
music_title=re.sub('[\\%s]'%('\\'.join(invalid_chars)),'_',music_title)
table=tree.xpath('/html/body/div[@class="status5"]/div[@class="data_body"]/table/tr')
ranking_data[ranking_type]=[]
for tr in table:
place=tr.xpath('td[1]/text()')
if place==[]: place=int(tr.xpath('td[1]/img/@src')[0].split('/ranking')[1][0])
else:place=int(place[0])
username=tr.xpath('td[3]/text()')[0].strip()
achievement_rate=float(tr.xpath('td[4]/text()')[0].strip()[:-1])
score=int(tr.xpath('td[5]/text()')[0].strip().replace(',',''))
ranking_data[ranking_type].append({x: eval(x) for x in ['place','username','achievement_rate','score']})
open(i+'/%s_%s.json'%(filename_prefix,music_title),'w').write(json.dumps(ranking_data,indent=4,ensure_ascii=False).encode('utf8'))
Raw
maimainet_playlog.py
#!python2
#coding:utf8
#Usage:
#Install Python 2.7
#Execute `pip install requests lxml` in Powershell/cmd/Terminal
#Supply your own Sega ID & password in `username_password.txt`
#Run the script
#If errors occur, just re-run the script
import requests,json,os
from lxml import html
def check_if_ok(result):
if not result.ok:
print result.url,result.status_code,result.reason
exit()
def post(session_requests,post_url,post_data,referer_url):
useragent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
headers={'user-agent':useragent,'referer':referer_url}
result=session_requests.post(post_url,data=post_data,headers=headers)
check_if_ok(result)
return result
def get(session_requests,get_url,referer_url):
useragent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
headers={'user-agent':useragent,'referer':referer_url}
result=session_requests.get(get_url,headers=headers)
check_if_ok(result)
return result
def safe_get_xpath(tree,xpath):
tmp=tree.xpath(xpath)
if tmp==[]: return ''
return tmp[0]
session_requests=requests.session()
#login
print 'login'
username,password=open('username_password.txt').read().split('\n')
login_page_url='https://maimai-net.com/maimai-mobile/'
login_post_url='https://maimai-net.com/maimai-mobile/index/submit/'
login_post_data={
'segaId':username,
'passWd':password,
'x':0,
'y':0
}
login_result=post(session_requests,login_post_url,login_post_data,login_page_url)
#playlog
page=1
print 'playlog page',page
playlog=[]
playlog_page_url='https://maimai-net.com/maimai-mobile/playLog/'
playlog_page_result=get(session_requests,playlog_page_url,'https://maimai-net.com/maimai-mobile/home/')
while True:
if os.path.exists('playlog_page%d.json'%page):
page+=1
continue
playlog_page_result.encoding='utf8'
playlog_page_content=playlog_page_result.text
tree=html.fromstring(playlog_page_content)
status5=tree.xpath('/html/body/div[@class="status5"]')[0]
accordion=status5.xpath('div[@id="accordion"]')[0]
play_count_current_page=int(accordion.xpath('count(*)'))/2
for i in range(play_count_current_page):
print 'play'
play=accordion.xpath('ul[%d]'%(i+1))[0]
date_time,shop_name=accordion.xpath('h3[%d]/text()'%(i+1))
special_play_type=safe_get_xpath(play,'li[1]/div[@class="red text_c"]/text()')
other_players=play.xpath('li[1]/form/button/text()')
track_count_current_play=int(play.xpath('count(li)'))-1
tracks=[]
for j in range(track_count_current_play):
print 'track'
track=accordion.xpath("ul[%d]/li[%d]"%(i+1,j+2))[0]
playIdx=int(track.xpath('div[@class="clearfix text_r"]/form/input')[0].value)
playlog_detail_post_url='https://maimai-net.com/maimai-mobile/playLog/detail'
playlog_detail_post_data=dict(playIdx=playIdx)
playlog_detail_result=post(session_requests,playlog_detail_post_url,playlog_detail_post_data,playlog_page_url)
playlog_detail_result.encoding='utf8'
playlog_detail_content=playlog_detail_result.text
tree=html.fromstring(playlog_detail_content).xpath('/html/body/div[@class="status5"]/div/ul/li')[0]
track_number,position=tree.xpath('span[1]/text()')[0].rsplit(' ',1)
difficulty=tree.xpath('span[2]/text()')[0][1:-1]
title=tree.xpath('div[@class="playdata_music_title"]/text()')[0]
achievement_rate=float(tree.xpath('div[@class="result_icon_block3 text_c f_l"]/text()')[1].split()[0].split(u'\uff1a')[1][:-1])
total_stats=[int(x.replace(',','')) for x in tree.xpath('table[1]/tr[2]/*/text()')[1:]]
next_rank_text=tree.xpath('table[1]/tr[3]/td[2]/text()')[0]
if next_rank_text==u"−−−−−": next_rank=0
else: next_rank=int(next_rank_text.replace(',',''))
max_combo,notecount=map(int,tree.xpath('table[1]/tr[4]/td[2]/text()')[0].split(' / '))
detailed_stats={y[0]:[int(z.replace(',','')) for z in y[1:]] for y in [x.xpath('td/text()') for x in tree.xpath('table[2]/tr')]}
if title=='Link':
if notecount=={'EASY':57,'BASIC':124,'ADVANCE':174,'EXPERT':314,'MASTER':308}[difficulty]: title='Link(JOYPOLIS)'
else: title='Link(niconico)'
current_track={x:eval(x) for x in ['track_number','position','difficulty','title','achievement_rate','total_stats','next_rank','max_combo','detailed_stats']}
tracks.append(current_track)
play_dict={x:eval(x) for x in ['date_time','shop_name','special_play_type','other_players','tracks']}
playlog.append(play_dict)
open('playlog_page%d.json'%page,'w').write(json.dumps(playlog,indent=4,ensure_ascii=False).encode('utf8'))
if u'次へ' not in playlog_page_content: break
page+=1
print 'playlog page',page
playlog_page_result=get(session_requests,'https://maimai-net.com/maimai-mobile/playLog/next/',playlog_page_url)
playlog_page_url=playlog_page_result.url
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment