Spaces:
Sleeping
Sleeping
| import subprocess | |
| import sys | |
| import json | |
| import gradio as gr | |
| from bs4 import BeautifulSoup | |
| from urllib.parse import urljoin | |
| # ========================================================= | |
| # CONFIG | |
| # ========================================================= | |
| TIMEOUT = 45000 | |
| BASE_URL = ( | |
| "https://en.xchina.co/models/type-10/" | |
| "sort-debut/1.html" | |
| ) | |
| #TOTAL_PAGE = 148 | |
| TOTAL_PAGE = 4 | |
| # ========================================================= | |
| # INSTALL PLAYWRIGHT | |
| # ========================================================= | |
| def setup_playwright(): | |
| print("="*50) | |
| print("Checking Playwright") | |
| try: | |
| import playwright | |
| print("Playwright OK") | |
| except Exception: | |
| return False | |
| try: | |
| result = subprocess.run( | |
| [ | |
| sys.executable, | |
| "-m", | |
| "playwright", | |
| "install", | |
| "chromium" | |
| ], | |
| stdout=subprocess.PIPE, | |
| stderr=subprocess.STDOUT, | |
| text=True, | |
| timeout=300 | |
| ) | |
| print(result.stdout) | |
| return result.returncode == 0 | |
| except Exception as e: | |
| print(e) | |
| return False | |
| PLAYWRIGHT_READY = setup_playwright() | |
| # ========================================================= | |
| # NORMALIZE | |
| # ========================================================= | |
| def normalize_url(url): | |
| url = (url or "").strip() | |
| if not url: | |
| return "" | |
| if not url.startswith( | |
| ( | |
| "http://", | |
| "https://" | |
| ) | |
| ): | |
| url = "https://" + url | |
| return url | |
| # ========================================================= | |
| # CLEAN HTML | |
| # ========================================================= | |
| def clean_html( | |
| html, | |
| base_url | |
| ): | |
| soup = BeautifulSoup( | |
| html, | |
| "html.parser" | |
| ) | |
| for tag in soup.find_all( | |
| [ | |
| "script", | |
| "iframe", | |
| "noscript" | |
| ] | |
| ): | |
| tag.decompose() | |
| for a in soup.find_all("a"): | |
| href = a.get("href") | |
| if href: | |
| a["href"] = urljoin( | |
| base_url, | |
| href | |
| ) | |
| for img in soup.find_all("img"): | |
| src = ( | |
| img.get("src") | |
| or | |
| img.get("data-src") | |
| ) | |
| if src: | |
| img["src"] = urljoin( | |
| base_url, | |
| src | |
| ) | |
| return soup.prettify() | |
| # ========================================================= | |
| # PLAYWRIGHT RENDER | |
| # ========================================================= | |
| def render_html(url): | |
| if not PLAYWRIGHT_READY: | |
| return "" | |
| from playwright.sync_api import sync_playwright | |
| try: | |
| with sync_playwright() as p: | |
| browser = p.chromium.launch( | |
| headless=True, | |
| args=[ | |
| "--no-sandbox", | |
| "--disable-dev-shm-usage", | |
| "--disable-blink-features=AutomationControlled" | |
| ] | |
| ) | |
| context = browser.new_context( | |
| viewport={ | |
| "width":1440, | |
| "height":1000 | |
| }, | |
| user_agent=( | |
| "Mozilla/5.0 " | |
| "(Windows NT 10.0; Win64; x64) " | |
| "Chrome/130 Safari/537.36" | |
| ), | |
| locale="en-US" | |
| ) | |
| page = context.new_page() | |
| page.goto( | |
| url, | |
| wait_until="domcontentloaded", | |
| timeout=TIMEOUT | |
| ) | |
| try: | |
| page.wait_for_load_state( | |
| "networkidle", | |
| timeout=15000 | |
| ) | |
| except: | |
| pass | |
| page.wait_for_timeout( | |
| 3000 | |
| ) | |
| # auto scroll | |
| try: | |
| page.evaluate( | |
| """ | |
| async()=>{ | |
| let sleep = | |
| ms=>new Promise( | |
| r=>setTimeout(r,ms) | |
| ); | |
| let last=0; | |
| for(let i=0;i<20;i++){ | |
| let h=document.body.scrollHeight; | |
| window.scrollTo(0,h); | |
| await sleep(500); | |
| if(h===last) | |
| break; | |
| last=h; | |
| } | |
| window.scrollTo(0,0); | |
| } | |
| """ | |
| ) | |
| except: | |
| pass | |
| html = page.content() | |
| browser.close() | |
| return clean_html( | |
| html, | |
| url | |
| ) | |
| except Exception as e: | |
| print(e) | |
| return "" | |
| # ========================================================= | |
| # EXTRACT MODELS | |
| # ========================================================= | |
| def extract_models( | |
| html, | |
| base_url | |
| ): | |
| soup = BeautifulSoup( | |
| html, | |
| "html.parser" | |
| ) | |
| models = [] | |
| cards = soup.select( | |
| ".item.actor" | |
| ) | |
| for card in cards: | |
| name = "" | |
| image = "" | |
| href = "" | |
| work = "" | |
| # ------------------------- | |
| # NAME | |
| # ------------------------- | |
| text = card.select_one( | |
| ".text" | |
| ) | |
| if text: | |
| name = text.get_text( | |
| " ", | |
| strip=True | |
| ) | |
| if "(" in name: | |
| name = name.split( | |
| "(" | |
| )[0].strip() | |
| # ------------------------- | |
| # HREF | |
| # ------------------------- | |
| a = card.find( | |
| "a" | |
| ) | |
| if a and a.get( | |
| "href" | |
| ): | |
| href = urljoin( | |
| base_url, | |
| a["href"] | |
| ) | |
| # ------------------------- | |
| # IMAGE | |
| # ------------------------- | |
| img = card.find( | |
| "img" | |
| ) | |
| if img: | |
| image = ( | |
| img.get("src") | |
| or | |
| img.get("data-src") | |
| or | |
| "" | |
| ) | |
| image = urljoin( | |
| base_url, | |
| image | |
| ) | |
| # ------------------------- | |
| # WORKS | |
| # ------------------------- | |
| tags = card.select_one( | |
| ".tags" | |
| ) | |
| if tags: | |
| value = tags.get_text( | |
| " ", | |
| strip=True | |
| ) | |
| if "Works:" in value: | |
| work = value.replace( | |
| "Works:", | |
| "" | |
| ).strip() | |
| models.append( | |
| { | |
| "name": name, | |
| "image": image, | |
| "href": href, | |
| "work": work | |
| } | |
| ) | |
| return models | |
| # ========================================================= | |
| # REMOVE DUPLICATE | |
| # ========================================================= | |
| def remove_duplicate( | |
| data | |
| ): | |
| result = [] | |
| seen = set() | |
| for item in data: | |
| href = item.get( | |
| "href" | |
| ) | |
| if href not in seen: | |
| seen.add( | |
| href | |
| ) | |
| result.append( | |
| item | |
| ) | |
| return result | |
| # ========================================================= | |
| # CRAWL ALL 148 PAGE | |
| # ========================================================= | |
| def crawl_all(): | |
| all_models = [] | |
| for page in range( | |
| 1, | |
| TOTAL_PAGE + 1 | |
| ): | |
| url = BASE_URL.replace( | |
| "/1.html", | |
| f"/{page}.html" | |
| ) | |
| print( | |
| "SCAN PAGE:", | |
| page | |
| ) | |
| html = render_html( | |
| url | |
| ) | |
| if not html: | |
| continue | |
| models = extract_models( | |
| html, | |
| url | |
| ) | |
| print( | |
| "FOUND:", | |
| len(models) | |
| ) | |
| all_models.extend( | |
| models | |
| ) | |
| all_models = remove_duplicate( | |
| all_models | |
| ) | |
| return all_models | |
| # ========================================================= | |
| # SAVE JSON | |
| # ========================================================= | |
| def start_crawl(): | |
| models = crawl_all() | |
| with open( | |
| "models.json", | |
| "w", | |
| encoding="utf-8" | |
| ) as f: | |
| json.dump( | |
| models, | |
| f, | |
| indent=4, | |
| ensure_ascii=False | |
| ) | |
| return json.dumps( | |
| models, | |
| indent=4, | |
| ensure_ascii=False | |
| ) | |
| # ========================================================= | |
| # GRADIO UI | |
| # ========================================================= | |
| with gr.Blocks( | |
| title="Xchina Model Crawler" | |
| ) as demo: | |
| gr.Markdown( | |
| """ | |
| # 🌐 Model Crawler | |
| Tự động quét: | |
| - 148 trang model | |
| - Name | |
| - Image | |
| - Href | |
| - Works | |
| Xuất: | |
| models.json | |
| """ | |
| ) | |
| btn = gr.Button( | |
| "🚀 Start Crawl", | |
| variant="primary" | |
| ) | |
| output = gr.Code( | |
| label="JSON Result", | |
| language="json", | |
| lines=40 | |
| ) | |
| btn.click( | |
| fn=start_crawl, | |
| inputs=[], | |
| outputs=output | |
| ) | |
| # ========================================================= | |
| # RUN | |
| # ========================================================= | |
| if __name__ == "__main__": | |
| print( | |
| "🚀 Starting Gradio" | |
| ) | |
| print( | |
| "Playwright:", | |
| PLAYWRIGHT_READY | |
| ) | |
| demo.launch( | |
| server_name="0.0.0.0", | |
| server_port=7860 | |
| ) |