55 lines
1.4 KiB
Python
55 lines
1.4 KiB
Python
from scrapegraphai.graphs import SmartScraperGraph
|
|
|
|
graph_config = {
|
|
"llm": {
|
|
"model": "ollama/mistral-nemo:12b",
|
|
"model_tokens": 1024000,
|
|
|
|
# "model": "ollama/mistral:7b",
|
|
# "model_tokens": 32768,
|
|
|
|
# "model": "ollama/llama3.2:3b",
|
|
# "model_tokens": 131072, # ollama show llama3.2:latest
|
|
|
|
# "model": "ollama/qwen2:7b",
|
|
# "model_tokens": 32768, # ollama show qwen2:7b
|
|
|
|
"temperature": 0, # 更准确执行任务
|
|
"format": "json", # Ollama 需要显式指定格式
|
|
},
|
|
"embeddings": {
|
|
"model": "ollama/nomic-embed-text",
|
|
},
|
|
"verbose": True,
|
|
# "headless": False,
|
|
}
|
|
|
|
|
|
# #############################
|
|
prompt="List all article titles on the page"
|
|
prompt="列出页面上的所有文章标题"
|
|
source="https://www.aivi.fyi/"
|
|
|
|
# #############################
|
|
prompt="List all Stations on the page."
|
|
source="https://network.satnogs.org/stations/"
|
|
|
|
# #############################
|
|
# prompt="列出页面上的所有站点。"
|
|
prompt="列出页面上的所有站点以及站点信息"
|
|
# prompt="列出页面上的所有站点以及站点信息。antennas需要是一个数组。"
|
|
source=""
|
|
with open('source/satnogs.html', 'r', encoding='utf-8') as file:
|
|
source = file.read()
|
|
|
|
|
|
smart_scraper_graph = SmartScraperGraph(
|
|
prompt=prompt,
|
|
source=source,
|
|
config=graph_config
|
|
)
|
|
|
|
# Run the pipeline
|
|
result = smart_scraper_graph.run()
|
|
print(result)
|