547 lines
22 KiB
Python
547 lines
22 KiB
Python
import re
|
||
import asyncio
|
||
import logging
|
||
import json
|
||
import os
|
||
from collections import OrderedDict
|
||
from datetime import datetime, timedelta
|
||
import difflib
|
||
import hashlib
|
||
from urllib.parse import urlparse, urlunparse, parse_qs
|
||
|
||
# 检查 aiohttp 是否安装
|
||
try:
|
||
import aiohttp
|
||
except ImportError:
|
||
print("错误: 缺少必要的依赖库 'aiohttp'。")
|
||
print("请使用以下命令安装:")
|
||
print("pip install aiohttp")
|
||
import sys
|
||
sys.exit(1)
|
||
|
||
# 检查 config 是否存在
|
||
try:
|
||
import config
|
||
required_attrs = ['source_urls', 'epg_urls', 'announcements', 'url_blacklist', 'ip_version_priority']
|
||
for attr in required_attrs:
|
||
if not hasattr(config, attr):
|
||
raise AttributeError(f"配置文件缺少必要的属性: {attr}")
|
||
except ImportError:
|
||
print("错误: 找不到配置模块 'config.py'。")
|
||
print("请确保项目目录下有 config.py 文件，内容示例如下:")
|
||
print("""
|
||
# config.py 示例内容
|
||
source_urls = [
|
||
"https://example.com/source1.m3u",
|
||
"https://example.com/source2.m3u"
|
||
]
|
||
epg_urls = ["https://example.com/epg.xml"]
|
||
announcements = [
|
||
{
|
||
"channel": "公告",
|
||
"entries": [
|
||
{
|
||
"name": None,
|
||
"url": "https://example.com/notice",
|
||
"logo": "https://picsum.photos/100/100?random=1"
|
||
}
|
||
]
|
||
}
|
||
]
|
||
url_blacklist = []
|
||
ip_version_priority = "ipv4"
|
||
""")
|
||
import sys
|
||
sys.exit(1)
|
||
except AttributeError as e:
|
||
print(f"配置文件错误: {e}")
|
||
import sys
|
||
sys.exit(1)
|
||
|
||
# 日志记录，只记录错误信息
|
||
logging.basicConfig(level=logging.ERROR, format='%(asctime)s - %(levelname)s - %(message)s',
|
||
handlers=[logging.FileHandler("./live/function.log", "w", encoding="utf-8"), logging.StreamHandler()])
|
||
|
||
# 确保 live 文件夹存在
|
||
output_folder = "live"
|
||
if not os.path.exists(output_folder):
|
||
os.makedirs(output_folder)
|
||
|
||
# 缓存文件夹和文件
|
||
cache_folder = "./live/cache"
|
||
cache_file = os.path.join(cache_folder, "url_cache.json")
|
||
cache_valid_days = 7 # 缓存有效期（天）
|
||
max_cache_entries = 500 # 最多保留的缓存条目数（防止极端膨胀）
|
||
|
||
if not os.path.exists(cache_folder):
|
||
os.makedirs(cache_folder)
|
||
|
||
def calculate_hash(content):
|
||
"""计算字符串的MD5哈希值"""
|
||
return hashlib.md5(content.encode('utf-8')).hexdigest()
|
||
|
||
def load_cache():
|
||
"""加载缓存文件，并确保其结构正确"""
|
||
if os.path.exists(cache_file):
|
||
try:
|
||
with open(cache_file, "r", encoding="utf-8") as f:
|
||
data = json.load(f)
|
||
# 兼容旧格式
|
||
if "urls" not in data:
|
||
data["urls"] = {}
|
||
if "timestamp" not in data:
|
||
data["timestamp"] = datetime.now().isoformat()
|
||
return data
|
||
except Exception as e:
|
||
logging.error(f"加载缓存失败: {e}")
|
||
return {"urls": {}, "timestamp": datetime.now().isoformat()}
|
||
|
||
def clean_expired_cache(cache, valid_hashes=None):
|
||
"""
|
||
清理过期或无效的缓存条目
|
||
- 移除 timestamp 超过 cache_valid_days 的条目
|
||
- 如果提供了 valid_hashes，则只保留哈希在该集合中的条目
|
||
- 限制最多保留 max_cache_entries 个条目（按最近使用时间排序）
|
||
"""
|
||
now = datetime.now()
|
||
urls = cache.get("urls", {})
|
||
cleaned = {}
|
||
|
||
for url_hash, entry in list(urls.items()):
|
||
try:
|
||
ts = datetime.fromisoformat(entry.get("timestamp", ""))
|
||
except Exception:
|
||
continue
|
||
if (now - ts).days >= cache_valid_days:
|
||
continue # 过期
|
||
if valid_hashes is not None and url_hash not in valid_hashes:
|
||
continue # 不在当前源URL列表中
|
||
cleaned[url_hash] = entry
|
||
|
||
# 如果仍过多，按时间戳保留最近的 max_cache_entries 条
|
||
if len(cleaned) > max_cache_entries:
|
||
sorted_entries = sorted(cleaned.items(),
|
||
key=lambda x: x[1].get("timestamp", ""),
|
||
reverse=True)
|
||
cleaned = dict(sorted_entries[:max_cache_entries])
|
||
|
||
cache["urls"] = cleaned
|
||
cache["timestamp"] = now.isoformat()
|
||
|
||
def save_cache(cache, valid_hashes=None):
|
||
"""保存缓存到磁盘，自动清理过期/无效条目"""
|
||
clean_expired_cache(cache, valid_hashes)
|
||
try:
|
||
with open(cache_file, "w", encoding="utf-8") as f:
|
||
json.dump(cache, f, ensure_ascii=False, indent=2)
|
||
except Exception as e:
|
||
logging.error(f"保存缓存失败: {e}")
|
||
|
||
def is_cache_valid(cache):
|
||
"""整体检查缓存文件的时间戳是否在有效期内（作为快速判断）"""
|
||
if not cache:
|
||
return False
|
||
try:
|
||
ts = datetime.fromisoformat(cache.get("timestamp", datetime.now().isoformat()))
|
||
return (datetime.now() - ts).days < cache_valid_days
|
||
except Exception:
|
||
return False
|
||
|
||
def parse_template(template_file):
|
||
"""解析 demo.txt 模板，返回 OrderedDict {类别: [频道名1, 频道名2...]} """
|
||
template_channels = OrderedDict()
|
||
current_category = None
|
||
|
||
with open(template_file, "r", encoding="utf-8") as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if line and not line.startswith("#"):
|
||
if "#genre#" in line:
|
||
current_category = line.split(",")[0].strip()
|
||
template_channels[current_category] = []
|
||
elif current_category:
|
||
channel_name = line.split(",")[0].strip()
|
||
template_channels[current_category].append(channel_name)
|
||
|
||
return template_channels
|
||
|
||
def clean_channel_name(channel_name):
|
||
"""清洗频道名称：去除特殊符号、空格，并将数字部分标准化"""
|
||
cleaned_name = re.sub(r'[$「」-]', '', channel_name)
|
||
cleaned_name = re.sub(r'\s+', '', cleaned_name)
|
||
# 将 "CCTV1" 转换为 "CCTV1"，但将 "CCTV10" 处理为 "CCTV10" 等（去掉前导零）
|
||
cleaned_name = re.sub(r'(\D*)(\d+)', lambda m: m.group(1) + str(int(m.group(2))), cleaned_name)
|
||
return cleaned_name.upper()
|
||
|
||
def is_valid_url(url):
|
||
return bool(re.match(r'^https?://', url))
|
||
|
||
def remove_unnecessary_params(url):
|
||
"""移除URL中不必要的查询参数（当前仅示例，可按需修改）"""
|
||
parsed_url = urlparse(url)
|
||
query_params = parse_qs(parsed_url.query)
|
||
# 保留必要参数（请按实际需要修改）
|
||
necessary_params = {}
|
||
for param, values in query_params.items():
|
||
if param in ['必要参数1', '必要参数2']:
|
||
necessary_params[param] = values
|
||
new_query = '&'.join([f'{param}={value[0]}' for param, value in necessary_params.items()])
|
||
new_url = urlunparse((parsed_url.scheme, parsed_url.netloc, parsed_url.path,
|
||
parsed_url.params, new_query, parsed_url.fragment))
|
||
return new_url
|
||
|
||
async def fetch_channels(session, url, cache, valid_source_hashes):
|
||
"""
|
||
异步获取指定URL的频道列表，优先从缓存读取
|
||
valid_source_hashes: 当前所有源URL的哈希集合，用于保存缓存时清理无效条目
|
||
"""
|
||
channels = OrderedDict()
|
||
unique_urls = set()
|
||
cache_hit = False
|
||
|
||
url_hash = calculate_hash(url)
|
||
# 检查具体的缓存条目是否有效
|
||
if url_hash in cache.get("urls", {}):
|
||
cached_entry = cache["urls"][url_hash]
|
||
try:
|
||
entry_time = datetime.fromisoformat(cached_entry.get("timestamp", ""))
|
||
if datetime.now() - entry_time <= timedelta(days=cache_valid_days):
|
||
logging.info(f"从缓存加载: {url}")
|
||
channels = OrderedDict(cached_entry.get("channels", {}))
|
||
unique_urls = set(cached_entry.get("unique_urls", []))
|
||
cache_hit = True
|
||
except Exception:
|
||
pass
|
||
|
||
if not cache_hit:
|
||
try:
|
||
async with session.get(url, timeout=aiohttp.ClientTimeout(total=30)) as response:
|
||
response.raise_for_status()
|
||
content = await response.text()
|
||
response.encoding = 'utf-8'
|
||
lines = content.split("\n")
|
||
# 判断文件类型
|
||
is_m3u = any(line.startswith("#EXTINF") for line in lines[:15])
|
||
if is_m3u:
|
||
channels.update(parse_m3u_lines(lines, unique_urls))
|
||
else:
|
||
channels.update(parse_txt_lines(lines, unique_urls))
|
||
|
||
if channels:
|
||
# 更新缓存
|
||
cache["urls"][url_hash] = {
|
||
"url": url,
|
||
"channels": dict(channels),
|
||
"unique_urls": list(unique_urls),
|
||
"timestamp": datetime.now().isoformat(),
|
||
"content_hash": calculate_hash(content)
|
||
}
|
||
# 每次更新后保存，并自动清理过期/无效条目
|
||
save_cache(cache, valid_hashes=valid_source_hashes)
|
||
|
||
except asyncio.TimeoutError:
|
||
logging.error(f"url: {url} 请求超时")
|
||
except Exception as e:
|
||
logging.error(f"url: {url} 失败❌, Error: {e}")
|
||
|
||
# 再次去重（同一频道同一URL）
|
||
for category, channel_list in channels.items():
|
||
seen = set()
|
||
unique_list = []
|
||
for ch_name, ch_url in channel_list:
|
||
if (ch_name, ch_url) not in seen:
|
||
unique_list.append((ch_name, ch_url))
|
||
seen.add((ch_name, ch_url))
|
||
channels[category] = unique_list
|
||
|
||
return channels
|
||
|
||
def parse_m3u_lines(lines, unique_urls):
|
||
"""解析 M3U 格式"""
|
||
channels = OrderedDict()
|
||
current_category = None
|
||
channel_name = None
|
||
|
||
for line in lines:
|
||
line = line.strip()
|
||
if line.startswith("#EXTINF"):
|
||
match = re.search(r'group-title="(.*?)",(.*)', line)
|
||
if match:
|
||
current_category = match.group(1).strip()
|
||
channel_name = match.group(2).strip()
|
||
if channel_name and channel_name.startswith("CCTV"):
|
||
channel_name = clean_channel_name(channel_name)
|
||
if current_category not in channels:
|
||
channels[current_category] = []
|
||
elif line and not line.startswith("#"):
|
||
channel_url = line.strip()
|
||
if is_valid_url(channel_url) and channel_url not in unique_urls:
|
||
unique_urls.add(channel_url)
|
||
if current_category and channel_name:
|
||
channels[current_category].append((channel_name, channel_url))
|
||
return channels
|
||
|
||
def parse_txt_lines(lines, unique_urls):
|
||
"""解析 TXT 格式（#genre#）"""
|
||
channels = OrderedDict()
|
||
current_category = None
|
||
|
||
for line in lines:
|
||
line = line.strip()
|
||
if "#genre#" in line:
|
||
current_category = line.split(",")[0].strip()
|
||
channels[current_category] = []
|
||
elif current_category:
|
||
match = re.match(r"^(.*?),(.*?)$", line)
|
||
if match:
|
||
channel_name = match.group(1).strip()
|
||
if channel_name and channel_name.startswith("CCTV"):
|
||
channel_name = clean_channel_name(channel_name)
|
||
channel_urls = match.group(2).strip().split('#')
|
||
|
||
for channel_url in channel_urls:
|
||
channel_url = channel_url.strip()
|
||
if is_valid_url(channel_url) and channel_url not in unique_urls:
|
||
unique_urls.add(channel_url)
|
||
channels[current_category].append((channel_name, channel_url))
|
||
else:
|
||
# 无逗号的行（纯URL），保留频道名为空
|
||
if is_valid_url(line) and line not in unique_urls:
|
||
unique_urls.add(line)
|
||
channels[current_category].append(("", line))
|
||
return channels
|
||
|
||
def find_similar_name(target_name, name_list):
|
||
"""在名称列表中寻找最相似的名字"""
|
||
matches = difflib.get_close_matches(target_name, name_list, n=1, cutoff=0.6)
|
||
return matches[0] if matches else None
|
||
|
||
async def filter_source_urls(template_file):
|
||
"""核心流程：获取所有源，匹配模板频道"""
|
||
template_channels = parse_template(template_file)
|
||
source_urls = config.source_urls
|
||
cache = load_cache()
|
||
|
||
# 计算当前源URL集合的哈希值（用于后续清理缓存）
|
||
valid_source_hashes = {calculate_hash(url) for url in source_urls}
|
||
|
||
all_channels = OrderedDict()
|
||
async with aiohttp.ClientSession() as session:
|
||
tasks = [fetch_channels(session, url, cache, valid_source_hashes) for url in source_urls]
|
||
fetched_channels_list = await asyncio.gather(*tasks)
|
||
|
||
for fetched in fetched_channels_list:
|
||
merge_channels(all_channels, fetched)
|
||
|
||
matched_channels = match_channels(template_channels, all_channels)
|
||
|
||
# 最终保存一次，清除所有过期或无效条目
|
||
save_cache(cache, valid_hashes=valid_source_hashes)
|
||
return matched_channels, template_channels, cache
|
||
|
||
def match_channels(template_channels, all_channels):
|
||
"""将模板频道与在线频道进行模糊匹配"""
|
||
matched_channels = OrderedDict()
|
||
|
||
# 收集所有在线频道名称（原始名）
|
||
all_online_names = []
|
||
for cat, chan_list in all_channels.items():
|
||
for ch_name, _ in chan_list:
|
||
all_online_names.append(ch_name)
|
||
|
||
# 清洗后的名字用于匹配
|
||
cleaned_online_names = [clean_channel_name(name) for name in all_online_names]
|
||
|
||
for category, channel_list in template_channels.items():
|
||
matched_channels[category] = OrderedDict()
|
||
for template_name in channel_list:
|
||
cleaned_template = clean_channel_name(template_name)
|
||
similar = find_similar_name(cleaned_template, cleaned_online_names)
|
||
if similar:
|
||
# 反查原始名称
|
||
original = next((name for name, cname in zip(all_online_names, cleaned_online_names) if cname == similar), None)
|
||
if original:
|
||
for online_cat, online_list in all_channels.items():
|
||
for ch_name, ch_url in online_list:
|
||
if ch_name == original:
|
||
matched_channels[category].setdefault(template_name, []).append(ch_url)
|
||
return matched_channels
|
||
|
||
def merge_channels(target, source):
|
||
"""合并频道字典"""
|
||
for category, channel_list in source.items():
|
||
if category in target:
|
||
target[category].extend(channel_list)
|
||
else:
|
||
target[category] = channel_list
|
||
|
||
def is_ipv6(url):
|
||
"""判断是否为 IPv6 地址格式"""
|
||
return re.match(r'^http:\/\/\[[0-9a-fA-F:]+\]', url) is not None
|
||
|
||
def updateChannelUrlsM3U(channels, template_channels, cache):
|
||
"""生成最终的 IPv4/IPv6 M3U 和 TXT 文件"""
|
||
written_urls_ipv4 = set()
|
||
written_urls_ipv6 = set()
|
||
url_changes = {"added": [], "removed": [], "modified": []}
|
||
|
||
# 检测URL变化（基于缓存中的历史数据）
|
||
if is_cache_valid(cache):
|
||
previous_urls = {}
|
||
for entry in cache.get("urls", {}).values():
|
||
for cat, chan_list in entry.get("channels", {}).items():
|
||
for ch_name, url in chan_list:
|
||
previous_urls[url] = (cat, ch_name)
|
||
|
||
current_urls = {}
|
||
for category, chan_dict in channels.items():
|
||
for ch_name, urls in chan_dict.items():
|
||
for url in urls:
|
||
current_urls[url] = (category, ch_name)
|
||
|
||
for url, (cat, name) in current_urls.items():
|
||
if url not in previous_urls:
|
||
url_changes["added"].append((cat, name, url))
|
||
|
||
for url, (cat, name) in previous_urls.items():
|
||
if url not in current_urls:
|
||
url_changes["removed"].append((cat, name, url))
|
||
|
||
# 更新公告中的日期占位符
|
||
current_date = datetime.now().strftime("%Y-%m-%d")
|
||
for group in config.announcements:
|
||
for announcement in group['entries']:
|
||
if announcement['name'] is None:
|
||
announcement['name'] = current_date
|
||
|
||
# 准备输出文件路径
|
||
ipv4_m3u = os.path.join(output_folder, "live_ipv4.m3u")
|
||
ipv4_txt = os.path.join(output_folder, "live_ipv4.txt")
|
||
ipv6_m3u = os.path.join(output_folder, "live_ipv6.m3u")
|
||
ipv6_txt = os.path.join(output_folder, "live_ipv6.txt")
|
||
|
||
with open(ipv4_m3u, "w", encoding="utf-8") as f_m3u_v4, \
|
||
open(ipv4_txt, "w", encoding="utf-8") as f_txt_v4, \
|
||
open(ipv6_m3u, "w", encoding="utf-8") as f_m3u_v6, \
|
||
open(ipv6_txt, "w", encoding="utf-8") as f_txt_v6:
|
||
|
||
# 写入 M3U 头部
|
||
epg_urls_str = ",".join(f'"{u}"' for u in config.epg_urls)
|
||
f_m3u_v4.write(f"#EXTM3U x-tvg-url={epg_urls_str}\n")
|
||
f_m3u_v6.write(f"#EXTM3U x-tvg-url={epg_urls_str}\n")
|
||
|
||
# 写入公告频道
|
||
for group in config.announcements:
|
||
f_txt_v4.write(f"{group['channel']},#genre#\n")
|
||
f_txt_v6.write(f"{group['channel']},#genre#\n")
|
||
for entry in group['entries']:
|
||
url = remove_unnecessary_params(entry['url'])
|
||
if is_ipv6(url):
|
||
if url not in written_urls_ipv6 and is_valid_url(url):
|
||
written_urls_ipv6.add(url)
|
||
write_to_files(f_m3u_v6, f_txt_v6, group['channel'], entry['name'], 1, url)
|
||
else:
|
||
if url not in written_urls_ipv4 and is_valid_url(url):
|
||
written_urls_ipv4.add(url)
|
||
write_to_files(f_m3u_v4, f_txt_v4, group['channel'], entry['name'], 1, url)
|
||
|
||
# 按模板顺序写入匹配的频道
|
||
for category, channel_list in template_channels.items():
|
||
f_txt_v4.write(f"{category},#genre#\n")
|
||
f_txt_v6.write(f"{category},#genre#\n")
|
||
if category in channels:
|
||
for ch_name in channel_list:
|
||
if ch_name in channels[category]:
|
||
sorted_v4 = []
|
||
sorted_v6 = []
|
||
for url in channels[category][ch_name]:
|
||
url = remove_unnecessary_params(url)
|
||
if is_ipv6(url):
|
||
if url not in written_urls_ipv6 and is_valid_url(url):
|
||
sorted_v6.append(url)
|
||
written_urls_ipv6.add(url)
|
||
else:
|
||
if url not in written_urls_ipv4 and is_valid_url(url):
|
||
sorted_v4.append(url)
|
||
written_urls_ipv4.add(url)
|
||
|
||
for idx, url in enumerate(sorted_v4, start=1):
|
||
new_url = add_url_suffix(url, idx, len(sorted_v4), "IPV4")
|
||
write_to_files(f_m3u_v4, f_txt_v4, category, ch_name, idx, new_url)
|
||
|
||
for idx, url in enumerate(sorted_v6, start=1):
|
||
new_url = add_url_suffix(url, idx, len(sorted_v6), "IPV6")
|
||
write_to_files(f_m3u_v6, f_txt_v6, category, ch_name, idx, new_url)
|
||
|
||
# TXT 文件末尾换行
|
||
f_txt_v4.write("\n")
|
||
f_txt_v6.write("\n")
|
||
|
||
# 记录URL变化日志
|
||
if any(url_changes.values()):
|
||
log_path = os.path.join(output_folder, "url_changes.log")
|
||
with open(log_path, "a", encoding="utf-8") as f:
|
||
f.write(f"\n=== 更新时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ===\n")
|
||
if url_changes["added"]:
|
||
f.write("\n新增URL:\n")
|
||
for cat, name, url in url_changes["added"]:
|
||
f.write(f"- {cat} - {name}: {url}\n")
|
||
if url_changes["removed"]:
|
||
f.write("\n移除URL:\n")
|
||
for cat, name, url in url_changes["removed"]:
|
||
f.write(f"- {cat} - {name}: {url}\n")
|
||
if url_changes["modified"]:
|
||
f.write("\n修改URL:\n")
|
||
for cat, name, old, new in url_changes["modified"]:
|
||
f.write(f"- {cat} - {name}: {old} → {new}\n")
|
||
|
||
def sort_and_filter_urls(urls, written_urls):
|
||
"""按 IP 优先级排序并过滤黑名单，此函数在原流程中未直接使用，但保留以备扩展"""
|
||
filtered = [
|
||
url for url in sorted(urls, key=lambda u: not is_ipv6(u) if config.ip_version_priority == "ipv6" else is_ipv6(u))
|
||
if url and url not in written_urls and not any(bl in url for bl in config.url_blacklist) and is_valid_url(url)
|
||
]
|
||
written_urls.update(filtered)
|
||
return filtered
|
||
|
||
def add_url_suffix(url, index, total, ip_version):
|
||
"""为URL添加线路标识后缀"""
|
||
suffix = f"${ip_version}" if total == 1 else f"${ip_version}•线路{index}"
|
||
base = url.split('$', 1)[0] if '$' in url else url
|
||
return f"{base}{suffix}"
|
||
|
||
def write_to_files(f_m3u, f_txt, category, channel_name, index, new_url):
|
||
"""写入M3U和TXT条目"""
|
||
f_m3u.write(f'#EXTINF:-1 group-title="{category}",{channel_name}\n')
|
||
f_m3u.write(new_url + "\n")
|
||
f_txt.write(f"{channel_name},{new_url}\n")
|
||
|
||
if __name__ == "__main__":
|
||
template_file = "demo.txt"
|
||
try:
|
||
if not os.path.exists(template_file):
|
||
print(f"错误: 找不到模板文件 '{template_file}'。")
|
||
print("请确保项目目录下有 demo.txt 文件。")
|
||
print("示例内容如下:")
|
||
print("""
|
||
# demo.txt 示例内容
|
||
央视,#genre#
|
||
CCTV-1
|
||
CCTV-2
|
||
卫视,#genre#
|
||
北京卫视
|
||
上海卫视
|
||
广东卫视
|
||
""")
|
||
import sys
|
||
sys.exit(1)
|
||
|
||
loop = asyncio.get_event_loop()
|
||
channels, template_channels, cache = loop.run_until_complete(filter_source_urls(template_file))
|
||
updateChannelUrlsM3U(channels, template_channels, cache)
|
||
loop.close()
|
||
print("操作完成！结果已保存到live文件夹。")
|
||
except Exception as e:
|
||
print(f"执行过程中发生错误: {e}")
|
||
logging.error(f"程序运行失败: {e}")
|