import requests
import pandas as pd
import json
import os
import re
import requests
# from loguru import logger
from bs4 import BeautifulSoup
from beeize.scraper import Scraper
import json
import os
import requests
from bs4 import BeautifulSoup
scraper = Scraper()


response = requests.get(url, headers=headers)
data = response.json()

news_list = data.get("data", {}).get("list", [])

news_data = []
for news in news_list:
    news_item = {
        "title": news.get("title", ""),
        "digest": news.get("digest", ""),
        "url": news.get("url", "")
    }
    scraper.push_data(news_item)
    print(news_item)

    news_data.append(news_item)

df = pd.DataFrame(news_data)
df.to_csv("news_data.csv", index=False, encoding="utf-8-sig")

print("News data has been saved to news_data.csv")