tanpengsccd/AV_Data_Capture
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #129 from jnozsc/fix_javdb

Browse Source 
fix javdb issue when it returns multiple results
This commit is contained in:
Yoshiko 2020-02-18 15:11:17 +08:00 committed by GitHub
commit 57cdd79003
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 38 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
javdb.py
View File

@ -59,9 +59,12 @@ def getTag(a):
result1 = str(html.xpath('//strong[contains(text(),"类别")]/../following-sibling::span/text()')).strip(" ['']") result1 = str(html.xpath('//strong[contains(text(),"类别")]/../following-sibling::span/text()')).strip(" ['']")
result2 = str(html.xpath('//strong[contains(text(),"类别")]/../following-sibling::span/a/text()')).strip(" ['']") result2 = str(html.xpath('//strong[contains(text(),"类别")]/../following-sibling::span/a/text()')).strip(" ['']")
return str(result1 + result2).strip('+').replace(",\\xa0", "").replace("'", "").replace(' ', '').replace(',,', '').lstrip(',') return str(result1 + result2).strip('+').replace(",\\xa0", "").replace("'", "").replace(' ', '').replace(',,', '').lstrip(',')
def getCover_small(a): def getCover_small(a, index=0):
# same issue mentioned below,
# javdb sometime returns multiple results
# DO NOT just get the firt one, get the one with correct index number
html = etree.fromstring(a, etree.HTMLParser()) # //table/tr[1]/td[1]/text() html = etree.fromstring(a, etree.HTMLParser()) # //table/tr[1]/td[1]/text()
result = html.xpath("//div[@class='item-image fix-scale-cover']/img/@src")[0] result = html.xpath("//div[@class='item-image fix-scale-cover']/img/@src")[index]
if not 'https' in result: if not 'https' in result:
result = 'https:' + result result = 'https:' + result
return result return result
@ -79,30 +82,39 @@ def getOutline(htmlcode):
result = str(html.xpath('//*[@id="introduction"]/dd/p[1]/text()')).strip(" ['']") result = str(html.xpath('//*[@id="introduction"]/dd/p[1]/text()')).strip(" ['']")
return result return result
def main(number): def main(number):
number = number.upper() try:
a = get_html('https://javdb.com/search?q=' + number + '&f=all') number = number.upper()
html = etree.fromstring(a, etree.HTMLParser()) # //table/tr[1]/td[1]/text() query_result = get_html('https://javdb.com/search?q=' + number + '&f=all')
result1 = html.xpath('//*[@id="videos"]/div/div/a/@href')[0] html = etree.fromstring(query_result, etree.HTMLParser()) # //table/tr[1]/td[1]/text()
b = get_html('https://javdb.com' + result1) # javdb sometime returns multiple results,
dic = { # and the first elememt maybe not the one we are looking for
'actor': getActor(b), # iterate all candidates and find the match one
'title': getTitle(b), urls = html.xpath('//*[@id="videos"]/div/div/a/@href')
'studio': getStudio(b), ids =html.xpath('//*[@id="videos"]/div/div/a/div[contains(@class, "uid")]/text()')
'outline': getOutline(b), correct_url = urls[ids.index(number)]
'runtime': getRuntime(b), detail_page = get_html('https://javdb.com' + correct_url)
'director': getDirector(b), dic = {
'release': getRelease(b), 'actor': getActor(detail_page),
'number': getNum(b), 'title': getTitle(detail_page),
'cover': getCover(b), 'studio': getStudio(detail_page),
'cover_small': getCover_small(a), 'outline': getOutline(detail_page),
'imagecut': 3, 'runtime': getRuntime(detail_page),
'tag': getTag(b), 'director': getDirector(detail_page),
'label': getLabel(b), 'release': getRelease(detail_page),
'year': getYear(getRelease(b)), # str(re.search('\d{4}',getRelease(a)).group()), 'number': getNum(detail_page),
'actor_photo': getActorPhoto(getActor(b)), 'cover': getCover(detail_page),
'website': 'https://javdb.com' + result1, 'cover_small': getCover_small(query_result, index=ids.index(number)),
'source': 'javdb.py', 'imagecut': 3,
} 'tag': getTag(detail_page),
'label': getLabel(detail_page),
'year': getYear(getRelease(detail_page)), # str(re.search('\d{4}',getRelease(a)).group()),
'actor_photo': getActorPhoto(getActor(detail_page)),
'website': 'https://javdb.com' + correct_url,
'source': 'javdb.py',
}
except Exception as e:
# print(e)
dic = {"title": ""}
js = json.dumps(dic, ensure_ascii=False, sort_keys=True, indent=4, separators=(',', ':'), ) # .encode('UTF-8') js = json.dumps(dic, ensure_ascii=False, sort_keys=True, indent=4, separators=(',', ':'), ) # .encode('UTF-8')
return js return js