Skip to content

Instantly share code, notes, and snippets.

@librz

librz/crawl_menu_urls.js

Last active October 8, 2023 09:10
Show Gist options
  • Save librz/d31d479bac3e966963a2bcb87c1920d0 to your computer and use it in GitHub Desktop.
Save librz/d31d479bac3e966963a2bcb87c1920d0 to your computer and use it in GitHub Desktop.
Download ZIP
Example of crawling 2-level deep menus & asynchronously get their URLs
Raw
crawl_menu_urls.js
/*
note: this is not a general purpose crawling script & can be only used as a reference
background:
1. target site has a side bar which contains many level-1 menus, each level-1 menu contains many level-2 menus
2. level-1 menu stores category name; level-2 menu stores system name, upon click will fire related event & load content as an iframe
*/
const sideMenu = document.querySelector("#side-menu");
const menus = Array.from(sideMenu.querySelectorAll(":scope > li:has(> ul.mm-collapse)"));
async function sleep(seconds) {
return new Promise((resolve) => {
setTimeout(() => {
resolve(null);
}, seconds * 1000);
});
}
async function getCategories() {
const beginTs = Date.now();
const collectIndex = {
menu: 0,
submenu: 0,
};
// collect categories
const categories = [];
for (const menu of menus) {
const category = {
category: menu.querySelector(":scope > a").textContent.trim(),
systems: [],
};
// collect systems
const submenus = Array.from(menu.querySelectorAll(":scope > ul.mm-collapse > li > a"));
const systems = [];
for (const submenu of submenus) {
const childNodes = Array.from(submenu.childNodes);
const textNodes = childNodes.filter((node) => node.nodeType === 3).filter((it) => it.textContent.trim());
if (textNodes.length > 1) {
console.warn(`More than one text node under menu #${collectIndex.menu}, submenu #${collectIndex.submenu}`);
}
const textNode = textNodes[0];
submenu.onclick();
await sleep(5); // wait for iframe to load (just to be safe)
const url = document.querySelector("iframe").src;
systems.push({
name: textNode.textContent.trim(),
url: url,
});
collectIndex.submenu += 1;
console.log(`Collected ${collectIndex.submenu} systems`);
}
category.systems = systems;
categories.push(category);
collectIndex.menu += 1;
console.log(`Collected ${collectIndex.menu} categories`);
}
// print time ellapsed
const endTs = Date.now();
const secondsEllapsed = Math.ceil((endTs - beginTs) / 1000);
console.log(`Finished in ${secondsEllapsed} seconds`);
// return
return categories;
}
getCategories().then(console.log);
Raw
results.json
[
{
"category": "OA",
"systems": [
{
"name": "OA办公系统",
"url": "https://www.yuque.com/pmmaster/tob_system/oabgxt?singleDoc#"
},
{
"name": "小微企业OA系统",
"url": "https://www.yuque.com/pmmaster/tob_system/xwqyoaxt?singleDoc#"
},
{
"name": "合同管理办公平台",
"url": "https://www.yuque.com/pmmaster/tob_system/htglbgpt?singleDoc#"
},
{
"name": "免费的OA办公系统",
"url": "https://www.yuque.com/pmmaster/tob_system/mfdoabgxt?singleDoc#"
},
{
"name": "开源OA协同办公平台",
"url": "https://www.yuque.com/pmmaster/tob_system/kyoaxtbgpt?singleDoc#"
},
{
"name": "医院OA系统",
"url": "https://www.yuque.com/pmmaster/tob_system/yyoaxt?singleDoc#"
},
{
"name": "企业OA办公系统",
"url": "https://www.yuque.com/pmmaster/tob_system/qyoabgxt?singleDoc#"
},
{
"name": "网络智能办公系统",
"url": "https://www.yuque.com/pmmaster/tob_system/wlznbgxt?singleDoc#"
},
{
"name": "企业通用OA办公平台",
"url": "https://www.yuque.com/pmmaster/tob_system/qytyoabgpt?singleDoc#"
},
{
"name": "企业数字化管理平台",
"url": "https://www.yuque.com/pmmaster/tob_system/qyszhglpt?singleDoc#"
}
]
},
{
"category": "IoT",
"systems": [
{
"name": "开源物联网平台",
"url": "https://www.yuque.com/pmmaster/tob_system/kywlwpt?singleDoc#"
},
{
"name": "物联网一体化平台",
"url": "https://www.yuque.com/pmmaster/tob_system/wlwythpt?singleDoc#"
},
{
"name": "物联网可视化平台",
"url": "https://www.yuque.com/pmmaster/tob_system/wlwkshpt?singleDoc#"
},
{
"name": "轻量级物联网设备接入平台",
"url": "https://www.yuque.com/pmmaster/tob_system/qljwlwsbjrpt?singleDoc#"
},
{
"name": "物联网规则引擎",
"url": "https://www.yuque.com/pmmaster/tob_system/wlwgzyq?singleDoc#"
},
{
"name": "物联网基础平台",
"url": "https://www.yuque.com/pmmaster/tob_system/wlwjcpt?singleDoc#"
},
{
"name": "开源的物联网开发平台",
"url": "https://www.yuque.com/pmmaster/tob_system/kydwlwkfpt?singleDoc#"
},
{
"name": "物联网基础开发平台",
"url": "https://www.yuque.com/pmmaster/tob_system/wlwjckfpt?singleDoc#"
},
{
"name": "物联网卡运营综合平台",
"url": "https://www.yuque.com/pmmaster/tob_system/wlwkyyzhpt?singleDoc#"
},
{
"name": "轻量级的物联网业务平台",
"url": "https://www.yuque.com/pmmaster/tob_system/qljdwlwywpt?singleDoc#"
},
{
"name": "超轻量级物联网开发平台",
"url": "https://www.yuque.com/pmmaster/tob_system/cqljwlwkfpt?singleDoc#"
},
{
"name": "企业级物联网基础开发平台",
"url": "https://www.yuque.com/pmmaster/tob_system/qyjwlwjckfpt?singleDoc#"
}
]
},
{
"category": "ERP",
"systems": [
{
"name": "人气领先的国产ERP系统",
"url": "https://www.yuque.com/pmmaster/tob_system/rqlxdgcerpxt?singleDoc#"
},
{
"name": "跨境ERP系统",
"url": "https://www.yuque.com/pmmaster/tob_system/kjerpxt?singleDoc#"
},
{
"name": "企业级ERP系统",
"url": "https://www.yuque.com/pmmaster/tob_system/qyjerpxt?singleDoc#"
}
]
},
{
"category": "MES",
"systems": [
{
"name": "零部件离散型MES系统",
"url": "https://www.yuque.com/pmmaster/tob_system/lbjlsxmesxt?singleDoc#"
},
{
"name": "开源的生产执行管理系统",
"url": "https://www.yuque.com/pmmaster/tob_system/kydsczxglxt?singleDoc#"
},
{
"name": "工厂管理助手MES管理系统",
"url": "https://www.yuque.com/pmmaster/tob_system/gcglzsmesglxt?singleDoc#"
},
{
"name": "生产执行管理MES系统",
"url": "https://www.yuque.com/pmmaster/tob_system/sczxglmesxt?singleDoc#"
},
{
"name": "开源能源管理系统",
"url": "https://www.yuque.com/pmmaster/tob_system/kynyglxt?singleDoc#"
}
]
},
{
"category": "TMS",
"systems": [
{
"name": "运输管理系统",
"url": "https://www.yuque.com/pmmaster/tob_system/ysglxt?singleDoc#"
},
{
"name": "物流管理平台",
"url": "https://www.yuque.com/pmmaster/tob_system/wlglpt?singleDoc#"
}
]
},
{
"category": "WMS",
"systems": [
{
"name": "开源的仓库管理系统WMS",
"url": "https://www.yuque.com/pmmaster/tob_system/kydckglxtwms?singleDoc#"
},
{
"name": "云仓仓储管理系统",
"url": "https://www.yuque.com/pmmaster/tob_system/ycccglxt?singleDoc#"
},
{
"name": "简单的仓储管理系统",
"url": "https://www.yuque.com/pmmaster/tob_system/jddccglxt?singleDoc#"
},
{
"name": "海外物流仓储管理系统",
"url": "https://www.yuque.com/pmmaster/tob_system/hwwlccglxt?singleDoc#"
}
]
},
{
"category": "CRM",
"systems": [
{
"name": "食品CRM",
"url": "https://www.yuque.com/pmmaster/tob_system/spcrm?singleDoc#"
},
{
"name": "零售CRM旗舰版",
"url": "https://www.yuque.com/pmmaster/tob_system/lscrmqjb?singleDoc#"
},
{
"name": "教堂CRM客户管理系统",
"url": "https://www.yuque.com/pmmaster/tob_system/jtcrmkhglxt?singleDoc#"
},
{
"name": "开源CRM系统",
"url": "https://www.yuque.com/pmmaster/tob_system/kycrmxt?singleDoc#"
},
{
"name": "CRM客户管理系统",
"url": "https://www.yuque.com/pmmaster/tob_system/crmkhglxt?singleDoc#"
},
{
"name": "开源CRM客户管理系统",
"url": "https://www.yuque.com/pmmaster/tob_system/kycrmkhglxt?singleDoc#"
},
{
"name": "企业CRM客户管理系统",
"url": "https://www.yuque.com/pmmaster/tob_system/qycrmkhglxt?singleDoc#"
},
{
"name": "千万级用户量的CRM系统",
"url": "https://www.yuque.com/pmmaster/tob_system/qwjyhldcrmxt?singleDoc#"
}
]
},
{
"category": "HRM",
"systems": [
{
"name": "人力资源平台",
"url": "https://www.yuque.com/pmmaster/tob_system/rlzypt?singleDoc#"
},
{
"name": "人力资源SaaS系统",
"url": "https://www.yuque.com/pmmaster/tob_system/rlzysaasxt?singleDoc#"
},
{
"name": "开源人力资源管理系统",
"url": "https://www.yuque.com/pmmaster/tob_system/kyrlzyglxt?singleDoc#"
}
]
},
{
"category": "SaaS",
"systems": [
{
"name": "电商SaaS系统",
"url": "https://www.yuque.com/pmmaster/tob_system/dssaasxt?singleDoc#"
},
{
"name": "人力资源SaaS系统",
"url": "https://www.yuque.com/pmmaster/tob_system/rlzysaasxt?singleDoc#"
}
]
},
{
"category": "OWTB",
"systems": [
{
"name": "订单仓库运输账单管理系统",
"url": "https://www.yuque.com/pmmaster/tob_system/ddckyszdglxt?singleDoc#"
}
]
},
{
"category": "SCRM",
"systems": [
{
"name": "企业微信私域SCRM系统",
"url": "https://www.yuque.com/pmmaster/tob_system/qywxsyscrmxt?singleDoc#"
},
{
"name": "企业微信的开源SCRM系统",
"url": "https://www.yuque.com/pmmaster/tob_system/qywxdkyscrmxt?singleDoc#"
}
]
},
{
"category": "DevOps",
"systems": [
{
"name": "云原生DevOps系统",
"url": "https://www.yuque.com/pmmaster/tob_system/yysdevopsxt?singleDoc#"
},
{
"name": "开源的云管理平台",
"url": "https://www.yuque.com/pmmaster/tob_system/kydyglpt?singleDoc#"
}
]
},
{
"category": "电商",
"systems": [
{
"name": "企业级开源商城系统",
"url": "https://www.yuque.com/pmmaster/tob_system/qyjkyscxt?singleDoc#"
},
{
"name": "企业级免费开源电商系统",
"url": "https://www.yuque.com/pmmaster/tob_system/qyjmfkydsxt?singleDoc#"
},
{
"name": "微商城电商系统",
"url": "https://www.yuque.com/pmmaster/tob_system/wscdsxt?singleDoc#"
},
{
"name": "单商户电商系统",
"url": "https://www.yuque.com/pmmaster/tob_system/dshdsxt?singleDoc#"
},
{
"name": "电商SaaS系统",
"url": "https://www.yuque.com/pmmaster/tob_system/dssaasxt?singleDoc#"
},
{
"name": "多商户电商系统",
"url": "https://www.yuque.com/pmmaster/tob_system/dshdsxt?singleDoc#"
},
{
"name": "单商户商城系统V3",
"url": "https://www.yuque.com/pmmaster/tob_system/dshscxtv3?singleDoc#"
},
{
"name": "单商户商城系统V5",
"url": "https://www.yuque.com/pmmaster/tob_system/dshscxtv5?singleDoc#"
},
{
"name": "开源跨境电商系统",
"url": "https://www.yuque.com/pmmaster/tob_system/kykjdsxt?singleDoc#"
},
{
"name": "B2B2C多用户商城系统",
"url": "https://www.yuque.com/pmmaster/tob_system/b2b2cdyhscxt?singleDoc#"
},
{
"name": "B2B2C多店铺商城系统",
"url": "https://www.yuque.com/pmmaster/tob_system/b2b2cddpscxt?singleDoc#"
},
{
"name": "B2B2C多商户商城系统V5",
"url": "https://www.yuque.com/pmmaster/tob_system/b2b2cdshscxtv5?singleDoc#"
}
]
},
{
"category": "零售",
"systems": [
{
"name": "零售收银系统",
"url": "https://www.yuque.com/pmmaster/tob_system/lssyxt?singleDoc#"
},
{
"name": "零售CRM旗舰版",
"url": "https://www.yuque.com/pmmaster/tob_system/lscrmqjb?singleDoc#"
},
{
"name": "新零售电商系统",
"url": "https://www.yuque.com/pmmaster/tob_system/xlsdsxt?singleDoc#"
},
{
"name": "新零售连锁店系统",
"url": "https://www.yuque.com/pmmaster/tob_system/xlslsdxt?singleDoc#"
},
{
"name": "新零售多门店系统",
"url": "https://www.yuque.com/pmmaster/tob_system/xlsdmdxt?singleDoc#"
}
]
},
{
"category": "营销",
"systems": [
{
"name": "数字化营销系统",
"url": "https://www.yuque.com/pmmaster/tob_system/szhyxxt?singleDoc#"
},
{
"name": "会员营销系统",
"url": "https://www.yuque.com/pmmaster/tob_system/hyyxxt?singleDoc#"
}
]
},
{
"category": "支付",
"systems": [
{
"name": "开源支付系统",
"url": "https://www.yuque.com/pmmaster/tob_system/kyzfxt?singleDoc#"
}
]
},
{
"category": "工具",
"systems": [
{
"name": "盲盒系统",
"url": "https://www.yuque.com/pmmaster/tob_system/mhxt?singleDoc#"
},
{
"name": "舆情系统",
"url": "https://www.yuque.com/pmmaster/tob_system/yqxt?singleDoc#"
},
{
"name": "商业智能开源版",
"url": "https://www.yuque.com/pmmaster/tob_system/syznkyb?singleDoc#"
},
{
"name": "知识文档管理系统",
"url": "https://www.yuque.com/pmmaster/tob_system/zswdglxt?singleDoc#"
}
]
},
{
"category": "教育",
"systems": [
{
"name": "教务系统",
"url": "https://www.yuque.com/pmmaster/tob_system/jwxt?singleDoc#"
},
{
"name": "在线培训考试系统",
"url": "https://www.yuque.com/pmmaster/tob_system/zxpxksxt?singleDoc#"
},
{
"name": "企业内部培训系统",
"url": "https://www.yuque.com/pmmaster/tob_system/qynbpxxt?singleDoc#"
},
{
"name": "教务管理系统单机构版",
"url": "https://www.yuque.com/pmmaster/tob_system/jwglxtdjgb?singleDoc#"
},
{
"name": "在线教育系统WEB开源版",
"url": "https://www.yuque.com/pmmaster/tob_system/zxjyxtwebkyb?singleDoc#"
}
]
},
{
"category": "财务",
"systems": [
{
"name": "餐饮专用财务系统",
"url": "https://www.yuque.com/pmmaster/tob_system/cyzycwxt?singleDoc#"
}
]
},
{
"category": "医疗",
"systems": [
{
"name": "医院排班系统",
"url": "https://www.yuque.com/pmmaster/tob_system/yypbxt?singleDoc#"
}
]
},
{
"category": "可视化",
"systems": [
{
"name": "Mars3D三维地球平台",
"url": "https://www.yuque.com/pmmaster/tob_system/mars3dswdqpt?singleDoc#"
},
{
"name": "数据可视化大屏设计器",
"url": "https://www.yuque.com/pmmaster/tob_system/sjkshdpsjq?singleDoc#"
},
{
"name": "web可视化工具",
"url": "https://www.yuque.com/pmmaster/tob_system/webkshgj?singleDoc#"
},
{
"name": "数据可视化平台",
"url": "https://www.yuque.com/pmmaster/tob_system/sjkshpt?singleDoc#"
},
{
"name": "自动化数据分析神器",
"url": "https://www.yuque.com/pmmaster/tob_system/zdhsjfxsq?singleDoc#"
},
{
"name": "简单易用的图表词典",
"url": "https://www.yuque.com/pmmaster/tob_system/jdyydtbcd?singleDoc#"
},
{
"name": "VeryReport报表系统",
"url": "https://www.yuque.com/pmmaster/tob_system/veryreportbbxt?singleDoc#"
}
]
},
{
"category": "低代码",
"systems": [
{
"name": "微办公低代码平台",
"url": "https://www.yuque.com/pmmaster/tob_system/wbgddmpt?singleDoc#"
},
{
"name": "企业级低代码平台",
"url": "https://www.yuque.com/pmmaster/tob_system/qyjddmpt?singleDoc#"
},
{
"name": "0代码页面搭建平台",
"url": "https://www.yuque.com/pmmaster/tob_system/0dmymdjpt?singleDoc#"
},
{
"name": "企业级快速开发平台",
"url": "https://www.yuque.com/pmmaster/tob_system/qyjkskfpt?singleDoc#"
},
{
"name": "企业管理低代码平台",
"url": "https://www.yuque.com/pmmaster/tob_system/qyglddmpt?singleDoc#"
},
{
"name": "可视化的前端低代码框架",
"url": "https://www.yuque.com/pmmaster/tob_system/kshdqdddmkj?singleDoc#"
}
]
},
{
"category": "进销存",
"systems": [
{
"name": "进销存ERP系统",
"url": "https://www.yuque.com/pmmaster/tob_system/jxcerpxt?singleDoc#"
},
{
"name": "ERP进销存系统",
"url": "https://www.yuque.com/pmmaster/tob_system/erpjxcxt?singleDoc#"
},
{
"name": "进销存开源版",
"url": "https://www.yuque.com/pmmaster/tob_system/jxckyb?singleDoc#"
},
{
"name": "中小企业ERP进销存系统",
"url": "https://www.yuque.com/pmmaster/tob_system/zxqyerpjxcxt?singleDoc#"
},
{
"name": "ERP系统.进销存.仓库管理",
"url": "https://www.yuque.com/pmmaster/tob_system/erpxt.jxc.ckgl?singleDoc#"
}
]
},
{
"category": "供应链",
"systems": [
{
"name": "供应链专业版",
"url": "https://www.yuque.com/pmmaster/tob_system/gylzyb?singleDoc#"
},
{
"name": "生鲜供应链中台",
"url": "https://www.yuque.com/pmmaster/tob_system/sxgylzt?singleDoc#"
}
]
},
{
"category": "数据中台",
"systems": [
{
"name": "数据中台系统",
"url": "https://www.yuque.com/pmmaster/tob_system/sjztxt?singleDoc#"
}
]
},
{
"category": "采购系统",
"systems": [
{
"name": "生鲜PDS采购配送",
"url": "https://www.yuque.com/pmmaster/tob_system/sxpdscgps?singleDoc#"
},
{
"name": "采购配送系统企业版",
"url": "https://www.yuque.com/pmmaster/tob_system/cgpsxtqyb?singleDoc#"
}
]
},
{
"category": "社区团购",
"systems": [
{
"name": "社区团购系统",
"url": "https://www.yuque.com/pmmaster/tob_system/sqtgxt?singleDoc#"
}
]
},
{
"category": "客服工单",
"systems": [
{
"name": "全文检索的工单系统",
"url": "https://www.yuque.com/pmmaster/tob_system/qwjsdgdxt?singleDoc#"
},
{
"name": "高效简易的工单系统",
"url": "https://www.yuque.com/pmmaster/tob_system/gxjydgdxt?singleDoc#"
},
{
"name": "工单管理系统",
"url": "https://www.yuque.com/pmmaster/tob_system/gdglxt?singleDoc#"
},
{
"name": "开源的客服系统",
"url": "https://www.yuque.com/pmmaster/tob_system/kydkfxt?singleDoc#"
},
{
"name": "智能AI客服系统",
"url": "https://www.yuque.com/pmmaster/tob_system/znaikfxt?singleDoc#"
},
{
"name": "开源客服工单系统",
"url": "https://www.yuque.com/pmmaster/tob_system/kykfgdxt?singleDoc#"
},
{
"name": "高性能的客户系统",
"url": "https://www.yuque.com/pmmaster/tob_system/gxndkhxt?singleDoc#"
}
]
},
{
"category": "智慧社区",
"systems": [
{
"name": "免费开源物业系统",
"url": "https://www.yuque.com/pmmaster/tob_system/mfkywyxt?singleDoc#"
},
{
"name": "智慧物业管理系统",
"url": "https://www.yuque.com/pmmaster/tob_system/zhwyglxt?singleDoc#"
}
]
},
{
"category": "能源管理",
"systems": [
{
"name": "开源能源管理系统",
"url": "https://www.yuque.com/pmmaster/tob_system/kynyglxt?singleDoc#"
}
]
},
{
"category": "资产管理",
"systems": [
{
"name": "EAM固定设备资产管理系统",
"url": "https://www.yuque.com/pmmaster/tob_system/eamgdsbzcglxt?singleDoc#"
}
]
},
{
"category": "项目管理",
"systems": [
{
"name": "项目研发管理工具",
"url": "https://www.yuque.com/pmmaster/tob_system/xmyfglgj?singleDoc#"
},
{
"name": "轻量级团队协作工具",
"url": "https://www.yuque.com/pmmaster/tob_system/qljtdxzgj?singleDoc#"
},
{
"name": "项目工时管理系统",
"url": "https://www.yuque.com/pmmaster/tob_system/xmgsglxt?singleDoc#"
},
{
"name": "项目管理团队协作平台",
"url": "https://www.yuque.com/pmmaster/tob_system/xmgltdxzpt?singleDoc#"
},
{
"name": "轻量级项目任务协作系统",
"url": "https://www.yuque.com/pmmaster/tob_system/qljxmrwxzxt?singleDoc#"
}
]
},
{
"category": "监控系统",
"systems": [
{
"name": "IT综合运维监控管理平台",
"url": "https://www.yuque.com/pmmaster/tob_system/itzhywjkglpt?singleDoc#"
},
{
"name": "一体化运维监控管理平台",
"url": "https://www.yuque.com/pmmaster/tob_system/ythywjkglpt?singleDoc#"
}
]
},
{
"category": "调度系统",
"systems": [
{
"name": "智能的任务调度框架",
"url": "https://www.yuque.com/pmmaster/tob_system/zndrwddkj?singleDoc#"
},
{
"name": "大数据调度服务监控平台",
"url": "https://www.yuque.com/pmmaster/tob_system/dsjddfwjkpt?singleDoc#"
}
]
},
{
"category": "流程表单引擎",
"systems": [
{
"name": "流程表单引擎-若依版",
"url": "https://www.yuque.com/pmmaster/tob_system/lcbdyq-ryb?singleDoc#"
},
{
"name": "流程表单引擎-集成版",
"url": "https://www.yuque.com/pmmaster/tob_system/lcbdyq-jcb?singleDoc#"
},
{
"name": "流程表单引擎-Vue3版",
"url": "https://www.yuque.com/pmmaster/tob_system/lcbdyq-vue3b?singleDoc#"
},
{
"name": "流程表单引擎-单组织版",
"url": "https://www.yuque.com/pmmaster/tob_system/lcbdyq-dzzb?singleDoc#"
},
{
"name": "流程表单引擎-Jeesite版",
"url": "https://www.yuque.com/pmmaster/tob_system/lcbdyq-jeesiteb?singleDoc#"
}
]
}
]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment