tanpengsccd/AV_Data_Capture
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Version 1.2 Update

Browse Source
This commit is contained in:
Yoshiko 2019-08-29 22:21:59 +08:00 committed by GitHub
parent 6928df8c3f
commit 2b9ce63601
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 90 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
ADC_function.py
View File

@ -1,3 +1,4 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import requests import requests
@ -18,18 +19,13 @@ if os.path.exists(config_file):
else: else:
print('[+]config.ini: not found, creating...') print('[+]config.ini: not found, creating...')
with open("config.ini", "wt", encoding='UTF-8') as code: with open("config.ini", "wt", encoding='UTF-8') as code:
print("[common]", file=code)
print("main_mode=1", file=code)
print("failed_output_folder=failed", file=code)
print("success_output_folder=JAV_output", file=code)
print("", file=code)
print("[proxy]",file=code) print("[proxy]",file=code)
print("proxy=127.0.0.1:1080",file=code) print("proxy=127.0.0.1:1080",file=code)
print("timeout=10", file=code) print("timeout=10", file=code)
print("retry=3", file=code) print("retry=3", file=code)
print("", file=code) print("", file=code)
print("[Name_Rule]", file=code) print("[Name_Rule]", file=code)
print("location_rule=actor+'/'+number",file=code) print("location_rule='JAV_output/'+actor+'/'+number",file=code)
print("naming_rule=number+'-'+title",file=code) print("naming_rule=number+'-'+title",file=code)
print("", file=code) print("", file=code)
print("[update]",file=code) print("[update]",file=code)
@ -41,8 +37,10 @@ else:
print("#plex only test!", file=code) print("#plex only test!", file=code)
print("", file=code) print("", file=code)
print("[directory_capture]", file=code) print("[directory_capture]", file=code)
print("switch=0", file=code)
print("directory=", file=code) print("directory=", file=code)
print("", file=code) print("", file=code)
print("everyone switch:1=on, 0=off", file=code)
time.sleep(2) time.sleep(2)
print('[+]config.ini: created!') print('[+]config.ini: created!')
try: try:

48
AV_Data_Capture.py
View File

@ -14,7 +14,7 @@ os.chdir(os.getcwd())
# ============global var=========== # ============global var===========
version='1.1' version='1.2'
config = ConfigParser() config = ConfigParser()
config.read(config_file, encoding='UTF-8') config.read(config_file, encoding='UTF-8')
@ -36,36 +36,26 @@ def UpdateCheck():
else: else:
print('[+]Update Check disabled!') print('[+]Update Check disabled!')
def movie_lists(): def movie_lists():
global exclude_directory_1
global exclude_directory_2
directory = config['directory_capture']['directory'] directory = config['directory_capture']['directory']
mp4=[] total=[]
avi=[] file_type = ['mp4','avi','rmvb','wmv','mov','mkv','flv','ts']
rmvb=[] exclude_directory_1 = config['common']['failed_output_folder']
wmv=[] exclude_directory_2 = config['common']['success_output_folder']
mov=[]
mkv=[]
flv=[]
ts=[]
if directory=='*': if directory=='*':
remove_total = []
for o in file_type:
remove_total += glob.glob(r"./" + exclude_directory_1 + "/*." + o)
remove_total += glob.glob(r"./" + exclude_directory_2 + "/*." + o)
for i in os.listdir(os.getcwd()): for i in os.listdir(os.getcwd()):
mp4 += glob.glob(r"./" + i + "/*.mp4") for a in file_type:
avi += glob.glob(r"./" + i + "/*.avi") total += glob.glob(r"./" + i + "/*." + a)
rmvb += glob.glob(r"./" + i + "/*.rmvb") for b in remove_total:
wmv += glob.glob(r"./" + i + "/*.wmv") total.remove(b)
mov += glob.glob(r"./" + i + "/*.mov")
mkv += glob.glob(r"./" + i + "/*.mkv")
flv += glob.glob(r"./" + i + "/*.flv")
ts += glob.glob(r"./" + i + "/*.ts")
total = mp4 + avi + rmvb + wmv + mov + mkv + flv + ts
return total return total
mp4 = glob.glob(r"./" + directory + "/*.mp4") for a in file_type:
avi = glob.glob(r"./" + directory + "/*.avi") total += glob.glob(r"./" + directory + "/*." + a)
rmvb = glob.glob(r"./" + directory + "/*.rmvb")
wmv = glob.glob(r"./" + directory + "/*.wmv")
mov = glob.glob(r"./" + directory + "/*.mov")
mkv = glob.glob(r"./" + directory + "/*.mkv")
flv = glob.glob(r"./" + directory + "/*.flv")
ts = glob.glob(r"./" + directory + "/*.ts")
total = mp4 + avi + rmvb + wmv + mov + mkv + flv + ts
return total return total
def CreatFailedFolder(): def CreatFailedFolder():
if not os.path.exists('failed/'): # 新建failed文件夹 if not os.path.exists('failed/'): # 新建failed文件夹
@ -160,7 +150,7 @@ if __name__ =='__main__':
shutil.move(i, str(os.getcwd()) + '/' + 'failed/') shutil.move(i, str(os.getcwd()) + '/' + 'failed/')
continue continue
CEF(exclude_directory_1)
CEF('JAV_output') CEF(exclude_directory_2)
print("[+]All finished!!!") print("[+]All finished!!!")
input("[+][+]Press enter key exit, you can check the error messge before you exit.\n[+][+]按回车键结束，你可以在结束之前查看和错误信息。") input("[+][+]Press enter key exit, you can check the error messge before you exit.\n[+][+]按回车键结束，你可以在结束之前查看和错误信息。")

67
core.py
View File

@ -18,14 +18,13 @@ import javbus
import javdb import javdb
#=========website======== #=========website========
#初始化全局变量
Config = ConfigParser() Config = ConfigParser()
Config.read(config_file, encoding='UTF-8') Config.read(config_file, encoding='UTF-8')
try: try:
option = ReadMediaWarehouse() option = ReadMediaWarehouse()
except: except:
print('[-]Config media_warehouse read failed!') print('[-]Config media_warehouse read failed!')
#初始化全局变量
title='' title=''
studio='' studio=''
year='' year=''
@ -54,6 +53,7 @@ program_mode = Config['common']['main_mode']
failed_folder= Config['common']['failed_output_folder'] failed_folder= Config['common']['failed_output_folder']
success_folder=Config['common']['success_output_folder'] success_folder=Config['common']['success_output_folder']
#=====================本地文件处理=========================== #=====================本地文件处理===========================
def moveFailedFolder(): def moveFailedFolder():
global filepath global filepath
print('[-]Move to Failed output folder') print('[-]Move to Failed output folder')
@ -100,37 +100,36 @@ def getDataFromJSON(file_number): #从JSON返回元数据
global naming_rule global naming_rule
global location_rule global location_rule
# ================================================网站规则添加开始================================================ # ================================================网站规则添加开始================================================
try: # 添加 需要 正则表达式的规则 if re.match('^\d{5,}', file_number):
if re.search('^\d{5,}', file_number).group() in file_number: json_data = json.loads(avsox.main(file_number))
json_data = json.loads(avsox.main(file_number)) if getDataState(json_data) == 0: # 如果元数据获取失败，请求番号至其他网站抓取
if getDataState(json_data) == 0: #如果元数据获取失败，请求番号至其他网站抓取 json_data = json.loads(javdb.main(file_number))
json_data = json.loads(javdb.main(file_number)) #==
elif re.match('\d+\D+', file_number):
elif re.search('\d+\D+', file_number).group() in file_number: json_data = json.loads(siro.main(file_number))
json_data = json.loads(siro.main(file_number)) if getDataState(json_data) == 0: # 如果元数据获取失败，请求番号至其他网站抓取
if getDataState(json_data) == 0: #如果元数据获取失败，请求番号至其他网站抓取
json_data = json.loads(javbus.main(file_number))
elif getDataState(json_data) == 0: #如果元数据获取失败，请求番号至其他网站抓取
json_data = json.loads(javdb.main(file_number))
except: # 添加 无需 正则表达式的规则
if 'fc2' in file_number:
json_data = json.loads(fc2fans_club.main(file_number.strip('fc2_').strip('fc2-').strip('ppv-').strip('PPV-').strip('FC2_').strip('FC2-').strip('ppv-').strip('PPV-')))
elif 'FC2' in file_number:
json_data = json.loads(fc2fans_club.main(file_number.strip('FC2_').strip('FC2-').strip('ppv-').strip('PPV-').strip('fc2_').strip('fc2-').strip('ppv-').strip('PPV-')))
elif 'HEYZO' in number or 'heyzo' in number or 'Heyzo' in number:
json_data = json.loads(avsox.main(file_number))
elif 'siro' in file_number or 'SIRO' in file_number or 'Siro' in file_number:
json_data = json.loads(siro.main(file_number))
else:
json_data = json.loads(javbus.main(file_number)) json_data = json.loads(javbus.main(file_number))
if getDataState(json_data) == 0: #如果元数据获取失败，请求番号至其他网站抓取 elif getDataState(json_data) == 0: # 如果元数据获取失败，请求番号至其他网站抓取
json_data = json.loads(avsox.main(file_number)) json_data = json.loads(javdb.main(file_number))
elif getDataState(json_data) == 0: #如果元数据获取失败，请求番号至其他网站抓取 # ==
json_data = json.loads(javdb.main(file_number)) elif 'fc2' in file_number or 'FC2' in file_number:
json_data = json.loads(fc2fans_club.main(
file_number))
# ==
elif 'HEYZO' in number or 'heyzo' in number or 'Heyzo' in number:
json_data = json.loads(avsox.main(file_number))
# ==
elif 'siro' in file_number or 'SIRO' in file_number or 'Siro' in file_number:
json_data = json.loads(siro.main(file_number))
# ==
else:
json_data = json.loads(javbus.main(file_number))
if getDataState(json_data) == 0: # 如果元数据获取失败，请求番号至其他网站抓取
json_data = json.loads(avsox.main(file_number))
elif getDataState(json_data) == 0: # 如果元数据获取失败，请求番号至其他网站抓取
json_data = json.loads(javdb.main(file_number))
# ================================================网站规则添加结束================================================ # ================================================网站规则添加结束================================================
@ -147,9 +146,9 @@ def getDataFromJSON(file_number): #从JSON返回元数据
try: try:
cover_small = json_data['cover_small'] cover_small = json_data['cover_small']
except: except:
aaaaaaa='' cover_small=''
imagecut = json_data['imagecut'] imagecut = json_data['imagecut']
tag = str(json_data['tag']).strip("[ ]").replace("'", '').replace(" ", '').split(',') # 字符串转列表 tag = str(json_data['tag']).strip("[ ]").replace("'", '').replace(" ", '').split(',') # 字符串转列表 @
actor = str(actor_list).strip("[ ]").replace("'", '').replace(" ", '') actor = str(actor_list).strip("[ ]").replace("'", '').replace(" ", '')
actor_photo = json_data['actor_photo'] actor_photo = json_data['actor_photo']
website = json_data['website'] website = json_data['website']
@ -324,12 +323,12 @@ def PrintFiles(filepath):
if cn_sub == '1': if cn_sub == '1':
print(" <tag>中文字幕</tag>", file=code) print(" <tag>中文字幕</tag>", file=code)
try: try:
for i in tag: for i in str(json_data['tag']).strip("[ ]").replace("'", '').replace(" ", '').split(','):
print(" <tag>" + i + "</tag>", file=code) print(" <tag>" + i + "</tag>", file=code)
except: except:
aaaaa = '' aaaaa = ''
try: try:
for i in tag: for i in str(json_data['tag']).strip("[ ]").replace("'", '').replace(" ", '').split(','):
print(" <genre>" + i + "</genre>", file=code) print(" <genre>" + i + "</genre>", file=code)
except: except:
aaaaaaaa = '' aaaaaaaa = ''

69
fc2fans_club.py
View File

@ -6,21 +6,23 @@ import ADC_function
def getTitle(htmlcode): #获取厂商 def getTitle(htmlcode): #获取厂商
#print(htmlcode) #print(htmlcode)
html = etree.fromstring(htmlcode,etree.HTMLParser()) html = etree.fromstring(htmlcode,etree.HTMLParser())
result = str(html.xpath('/html/body/div[2]/div/div[1]/h3/text()')).strip(" ['']") result = str(html.xpath('//*[@id="container"]/div[1]/div/article/section[1]/h2/text()')).strip(" ['']")
result2 = str(re.sub('\D{2}2-\d+','',result)).replace(' ','',1)
#print(result2) #print(result2)
return result2 return result
def getActor(htmlcode): def getActor(htmlcode):
try: try:
html = etree.fromstring(htmlcode, etree.HTMLParser()) html = etree.fromstring(htmlcode, etree.HTMLParser())
result = str(html.xpath('/html/body/div[2]/div/div[1]/h5[5]/a/text()')).strip(" ['']") result = str(html.xpath('//*[@id="container"]/div[1]/div/article/section[1]/div/div[2]/dl/dd[5]/a/text()')).strip(" ['']")
return result return result
except: except:
return '' return ''
def getStudio(htmlcode): #获取厂商 def getStudio(htmlcode): #获取厂商
html = etree.fromstring(htmlcode,etree.HTMLParser()) try:
result = str(html.xpath('/html/body/div[2]/div/div[1]/h5[3]/a[1]/text()')).strip(" ['']") html = etree.fromstring(htmlcode, etree.HTMLParser())
return result result = str(html.xpath('//*[@id="container"]/div[1]/div/article/section[1]/div/div[2]/dl/dd[5]/a/text()')).strip(" ['']")
return result
except:
return ''
def getNum(htmlcode): #获取番号 def getNum(htmlcode): #获取番号
html = etree.fromstring(htmlcode, etree.HTMLParser()) html = etree.fromstring(htmlcode, etree.HTMLParser())
result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[1]/span[2]/text()')).strip(" ['']") result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[1]/span[2]/text()')).strip(" ['']")
@ -31,24 +33,23 @@ def getRelease(htmlcode2): #
html=etree.fromstring(htmlcode2,etree.HTMLParser()) html=etree.fromstring(htmlcode2,etree.HTMLParser())
result = str(html.xpath('//*[@id="container"]/div[1]/div/article/section[1]/div/div[2]/dl/dd[4]/text()')).strip(" ['']") result = str(html.xpath('//*[@id="container"]/div[1]/div/article/section[1]/div/div[2]/dl/dd[4]/text()')).strip(" ['']")
return result return result
def getCover(htmlcode,number,htmlcode2): #获取厂商 # def getCover(htmlcode2): #获取厂商 #
#a = ADC_function.get_html('http://adult.contents.fc2.com/article_search.php?id=' + str(number).lstrip("FC2-").lstrip("fc2-").lstrip("fc2_").lstrip("fc2-") + '&utm_source=aff_php&utm_medium=source_code&utm_campaign=from_aff_php') #a = ADC_function.get_html('http://adult.contents.fc2.com/article_search.php?id=' + str(number).lstrip("FC2-").lstrip("fc2-").lstrip("fc2_").lstrip("fc2-") + '&utm_source=aff_php&utm_medium=source_code&utm_campaign=from_aff_php')
html = etree.fromstring(htmlcode2, etree.HTMLParser()) html = etree.fromstring(htmlcode2, etree.HTMLParser())
result = str(html.xpath('//*[@id="container"]/div[1]/div/article/section[1]/div/div[1]/a/img/@src')).strip(" ['']") result = str(html.xpath('//*[@id="container"]/div[1]/div/article/section[1]/div/div[1]/a/img/@src')).strip(" ['']")
if result == '': # if result == '':
html = etree.fromstring(htmlcode, etree.HTMLParser()) # html = etree.fromstring(htmlcode, etree.HTMLParser())
result2 = str(html.xpath('//*[@id="slider"]/ul[1]/li[1]/img/@src')).strip(" ['']") # result2 = str(html.xpath('//*[@id="slider"]/ul[1]/li[1]/img/@src')).strip(" ['']")
return 'http://fc2fans.club' + result2 # return result2
return 'http:' + result return 'http:' + result
def getOutline(htmlcode2,number): #获取番号 # def getOutline(htmlcode2): #获取番号 #
#a = ADC_function.get_html('http://adult.contents.fc2.com/article_search.php?id=' + str(number).lstrip("FC2-").lstrip("fc2-").lstrip("fc2_").lstrip("fc2-") + '&utm_source=aff_php&utm_medium=source_code&utm_campaign=from_aff_php')
html = etree.fromstring(htmlcode2, etree.HTMLParser()) html = etree.fromstring(htmlcode2, etree.HTMLParser())
result = str(html.xpath('//*[@id="container"]/div[1]/div/article/section[4]/p/text()')).replace("\\n",'',10000).strip(" ['']").replace("'",'',10000) result = str(html.xpath('//*[@id="container"]/div[1]/div/article/section[4]/p/text()')).strip(" ['']").replace("\\n",'',10000).replace("'",'',10000).replace(', ,','').strip(' ').replace('。,',',')
return result return result
def getTag(htmlcode): #获取番号 def getTag(htmlcode): #获取番号
html = etree.fromstring(htmlcode, etree.HTMLParser()) html = etree.fromstring(htmlcode, etree.HTMLParser())
result = str(html.xpath('/html/body/div[2]/div/div[1]/h5[4]/a/text()')) result = html.xpath('//*[@id="container"]/div[1]/div/article/section[6]/ul/li/a/text()')
return result.strip(" ['']").replace("'",'').replace(' ','') return result
def getYear(release): def getYear(release):
try: try:
result = re.search('\d{4}',release).group() result = re.search('\d{4}',release).group()
@ -56,30 +57,28 @@ def getYear(release):
except: except:
return '' return ''
def main(number2): def main(number):
number=number2.replace('PPV','').replace('ppv','') number=number.replace('PPV','').replace('ppv','').strip('fc2_').strip('fc2-').strip('ppv-').strip('PPV-').strip('FC2_').strip('FC2-').strip('ppv-').strip('PPV-')
htmlcode2 = ADC_function.get_html('http://adult.contents.fc2.com/article_search.php?id='+str(number).lstrip("FC2-").lstrip("fc2-").lstrip("fc2_").lstrip("fc2-")+'&utm_source=aff_php&utm_medium=source_code&utm_campaign=from_aff_php') htmlcode2 = ADC_function.get_html('http://adult.contents.fc2.com/article_search.php?id='+str(number).lstrip("FC2-").lstrip("fc2-").lstrip("fc2_").lstrip("fc2-")+'')
htmlcode = ADC_function.get_html('http://fc2fans.club/html/FC2-' + number + '.html') #htmlcode = ADC_function.get_html('http://fc2fans.club/html/FC2-' + number + '.html')
dic = { dic = {
'title': getTitle(htmlcode), 'title': getTitle(htmlcode2),
'studio': getStudio(htmlcode), 'studio': getStudio(htmlcode2),
'year': '',#str(re.search('\d{4}',getRelease(number)).group()), 'year': getYear(getRelease(htmlcode2)),
'outline': getOutline(htmlcode,number), 'outline': getOutline(htmlcode2),
'runtime': getYear(getRelease(htmlcode)), 'runtime': getYear(getRelease(htmlcode2)),
'director': getStudio(htmlcode), 'director': getStudio(htmlcode2),
'actor': getActor(htmlcode), 'actor': getStudio(htmlcode2),
'release': getRelease(number), 'release': getRelease(htmlcode2),
'number': 'FC2-'+number, 'number': 'FC2-'+number,
'cover': getCover(htmlcode,number,htmlcode2), 'cover': getCover(htmlcode2),
'imagecut': 0, 'imagecut': 0,
'tag': getTag(htmlcode), 'tag': getTag(htmlcode2),
'actor_photo':'', 'actor_photo':'',
'website': 'http://fc2fans.club/html/FC2-' + number + '.html', 'website': 'http://adult.contents.fc2.com/article_search.php?id=' + number,
'source': 'fc2fans_club.py', 'source': 'fc2fans_club.py',
} }
#print(getTitle(htmlcode))
#print(getNum(htmlcode))
js = json.dumps(dic, ensure_ascii=False, sort_keys=True, indent=4, separators=(',', ':'),)#.encode('UTF-8') js = json.dumps(dic, ensure_ascii=False, sort_keys=True, indent=4, separators=(',', ':'),)#.encode('UTF-8')
return js return js
#print(main('1051725')) #print(main('1145465'))