diff --git a/py/IPTV搜索引擎采集.py b/py/IPTV搜索引擎采集.py index 546e03d..f64107f 100644 --- a/py/IPTV搜索引擎采集.py +++ b/py/IPTV搜索引擎采集.py @@ -22,7 +22,139 @@ import cv2 from bs4 import BeautifulSoup from urllib.parse import urlparse from translate import Translator # 导入Translator类,用于文本翻译 +###################################################################################################################### +# 获取rtp目录下的文件名,组播IP采集 +files = os.listdir('rtp') +files_name = [] +# 去除后缀名并保存至provinces_isps +for file in files: + name, extension = os.path.splitext(file) + files_name.append(name) +#忽略不符合要求的文件名 +provinces_isps = [name for name in files_name if name.count('_') == 1] +print(f"本次查询:{provinces_isps}的组播节目") +keywords = [] +for province_isp in provinces_isps: + # 读取文件并删除空白行 + try: + with open(f'rtp/{province_isp}.txt', 'r', encoding='utf-8') as file: + lines = file.readlines() + lines = [line.strip() for line in lines if line.strip()] + # 获取第二行中以包含 "rtp://" 的值作为 mcast + if lines: + first_line = lines[1] + if "rtp://" in first_line: + mcast = first_line.split("rtp://")[1].split(" ")[0] + keywords.append(province_isp + "_" + mcast) + except FileNotFoundError: + # 如果文件不存在,则捕获 FileNotFoundError 异常并打印提示信息 + print(f"文件 '{province_isp}.txt' 不存在. 跳过此文件.") +for keyword in keywords: + province, isp, mcast = keyword.split("_") + #将省份转成英文小写 + # 根据不同的 isp 设置不同的 org 值 + if province == "北京" and isp == "联通": + isp_en = "cucc" + org = "China Unicom Beijing Province Network" + elif isp == "联通": + isp_en = "cucc" + org = "CHINA UNICOM China169 Backbone" + elif isp == "电信": + org = "Chinanet" + isp_en = "ctcc" + elif isp == "移动": + org == "China Mobile communications corporation" + isp_en = "cmcc" + + current_time = datetime.now() + timeout_cnt = 0 + result_urls = set() + while len(result_urls) == 0 and timeout_cnt <= 5: + try: + search_url = 'https://fofa.info/result?qbase64=' + search_txt = f'\"udpxy\" && country=\"CN\" && region=\"{province}\" && org=\"{org}\"' # && org=\"{org}\" + # 将字符串编码为字节流 + bytes_string = search_txt.encode('utf-8') + # 使用 base64 进行编码 + search_txt = base64.b64encode(bytes_string).decode('utf-8') + search_url += search_txt + print(f"{current_time} 查询运营商 : {province}{isp} ,查询网址 : {search_url}") + response = requests.get(search_url, timeout=5) + # 处理响应 + response.raise_for_status() + # 检查请求是否成功 + html_content = response.text + # 使用BeautifulSoup解析网页内容 + html_soup = BeautifulSoup(html_content, "html.parser") + # print(f"{current_time} html_content:{html_content}") + # 查找所有符合指定格式的网址 + # 设置匹配的格式,如http://8.8.8.8:8888 + pattern = r"http://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d+" + urls_all = re.findall(pattern, html_content) + # 去重得到唯一的URL列表 + result_urls = set(urls_all) + print(f"{current_time} result_urls:{result_urls}") + valid_ips = [] + # 遍历所有视频链接 + for url in result_urls: + video_url = url + "/rtp/" + mcast + # 用OpenCV读取视频 + cap = cv2.VideoCapture(video_url) + # 检查视频是否成功打开 + if not cap.isOpened(): + print(f"{current_time} {video_url} 无效") + else: + # 读取视频的宽度和高度 + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + print(f"{current_time} {video_url} 的分辨率为 {width}x{height}") + # 检查分辨率是否大于0 + if width > 0 and height > 0: + valid_ips.append(url) + # 关闭视频流 + cap.release() + + if valid_ips: + #生成节目列表 省份运营商.txt + rtp_filename = f'rtp/{province}_{isp}.txt' + with open(rtp_filename, 'r', encoding='utf-8') as file: + data = file.read() + txt_filename = f'playlist/{province}{isp}.txt' + with open(txt_filename, 'a') as new_file: #以追加形式写入 + for url in valid_ips: + new_data = data.replace("rtp://", f"{url}/rtp/") + new_file.write(new_data) + print(f'已生成播放列表,保存至{txt_filename}') + except (requests.Timeout, requests.RequestException) as e: + timeout_cnt += 1 + print(f"{current_time} [{province}]搜索请求发生超时,异常次数:{timeout_cnt}") + if timeout_cnt <= 5: + # 退出循环迭代 + break + else: + print(f"{current_time} 搜索IPTV频道源[],超时次数过多:{timeout_cnt} 次,停止处理") +print('节目表制作完成! 文件输出在playlist文件夹!') + +print('对playlist文件夹里面的所有txt文件进行去重处理') +def remove_duplicates_keep_order(folder_path): + for filename in os.listdir(folder_path): + if filename.endswith('.txt'): + file_path = os.path.join(folder_path, filename) + lines = set() + unique_lines = [] + with open(file_path, 'r', encoding='utf-8') as file: + for line in file: + if line not in lines: + unique_lines.append(line) + lines.add(line) + # 将保持顺序的去重后的内容写回原文件 + with open(file_path, 'w', encoding='utf-8') as file: + file.writelines(unique_lines) +# 使用示例 +folder_path = 'playlist' # 替换为你的文件夹路径 +remove_duplicates_keep_order(folder_path) +print('文件去重完成!移除存储的旧文件!') # 定义请求头 header = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"