Skip to content

Commit

Permalink
Merge pull request Kalanyr#15 from jpenney/skipfiles
Browse files Browse the repository at this point in the history 
add -skipfiles
  • Loading branch information
@Kalanyr
Kalanyr authored Jul 27, 2018
2 parents 8e4d45b + fb8b384 commit 78e8601
Showing 1 changed file with 40 additions and 9 deletions.
49 changes: 40 additions & 9 deletions gogrepoc.py
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import OpenSSL
import platform
import locale
from fnmatch import fnmatch
# python 2 / 3 imports
try:
# python 2
Expand Down Expand Up @@ -719,7 +720,14 @@ def deDuplicateName(potentialItem,clashDict):
#No Name Clash
clashDict[potentialItem.name] = [(potentialItem.md5,potentialItem.size)]
return potentialItem.name


def check_skip_file(fname, skipfiles):
# return pattern that matched, or None
for skipf in skipfiles:
if fnmatch(fname, skipf):
return skipf
return None

def process_path(path):
fpath = path
if sys.version_info[0] <= 2:
Expand Down Expand Up @@ -785,6 +793,7 @@ def process_argv(argv):
g3.add_argument('-ids', action='store', help='id(s) or title(s) of the game in the manifest to download', nargs='*', default=[])
g3.add_argument('-skipids', action='store', help='id(s) or title(s) of the game(s) in the manifest to NOT download', nargs='*', default=[])
g3.add_argument('-id', action='store', help='(deprecated) id or title of the game in the manifest to download')
g1.add_argument('-skipfiles', action='store', help='file name (or glob patterns) to NOT download', nargs='*', default=[])
g1.add_argument('-wait', action='store', type=float,
help='wait this long in hours before starting', default=0.0) # sleep in hr
g4 = g1.add_mutually_exclusive_group() # below are mutually exclusive
Expand Down Expand Up @@ -852,6 +861,7 @@ def process_argv(argv):
g3.add_argument('-ids', action='store', help='id(s) or title(s) of the game in the manifest to verify', nargs='*', default=[])
g3.add_argument('-skipids', action='store', help='id(s) or title(s) of the game[s] in the manifest to NOT verify', nargs='*', default=[])
g3.add_argument('-id', action='store', help='(deprecated) id or title of the game in the manifest to verify')
g1.add_argument('-skipfiles', action='store', help='file name (or glob patterns) to NOT verify', nargs='*', default=[])
g4 = g1.add_mutually_exclusive_group() # below are mutually exclusive
g4.add_argument('-skipos', action='store', help='skip verification of game files for operating system(s)', nargs='*', default=[])
g4.add_argument('-os', action='store', help='verify game files only for operating system(s)', nargs='*', default=[x for x in VALID_OS_TYPES])
Expand Down Expand Up @@ -905,6 +915,7 @@ def process_argv(argv):
return args



# --------
# Commands
# --------
Expand Down Expand Up @@ -1343,7 +1354,7 @@ def cmd_import(src_dir, dest_dir,os_list,lang_list,skipextras,skipids,ids,skipga
shutil.copy(f, dest_file)


def cmd_download(savedir, skipextras,skipids, dryrun, ids,os_list, lang_list,skipgalaxy,skipstandalone,skipshared):
def cmd_download(savedir, skipextras,skipids, dryrun, ids,os_list, lang_list,skipgalaxy,skipstandalone,skipshared, skipfiles):
sizes, rates, errors = {}, {}, {}
work = Queue() # build a list of work items

Expand All @@ -1370,6 +1381,10 @@ def gigs(b):
info("skipping games with id(s): {%s}" % formattedSkipIds)
downloadItems = [item for item in items if item.title not in skipids and str(item.id) not in skipids]
items = downloadItems

if skipfiles:
formattedSkipFiles = "'" + "', '".join(skipfiles) + "'"
info("skipping files that match: {%s}" % formattedSkipFiles)

if not items:
if ids and skipids:
Expand Down Expand Up @@ -1496,19 +1511,23 @@ def gigs(b):
item.serial = item.serial.replace(u'<span>', '')
item.serial = item.serial.replace(u'</span>', os.linesep)
fd_serial.write(item.serial)



# Populate queue with all files to be downloaded
for game_item in item.downloads + item.galaxyDownloads + item.sharedDownloads + item.extras:
if game_item.name is None:
continue # no game name, usually due to 404 during file fetch

skipfile_skip = check_skip_file(game_item.name, skipfiles)
if skipfile_skip:
info(' skip %s (matches "%s")' % (game_item.name, skipfile_skip))
continue

dest_file = os.path.join(item_homedir, game_item.name)
downloading_file = os.path.join(item_downloaddir, game_item.name)

if os.path.isfile(dest_file):
if game_item.size is None:
warn(' unknown %s has no size info. skipping')
warn(' unknown %s has no size info. skipping' % game_item.name)
continue
elif game_item.size != os.path.getsize(dest_file):
warn(' fail %s has incorrect size.' % game_item.name)
Expand Down Expand Up @@ -1944,7 +1963,7 @@ def cmd_backup(src_dir, dest_dir,skipextras,os_list,lang_list,ids,skipids,skipga
shutil.copy(os.path.join(src_game_dir, extra_file), dest_game_dir)


def cmd_verify(gamedir, skipextras, skipids, check_md5, check_filesize, check_zips, delete_on_fail, clean_on_fail, ids, os_list, lang_list, skipgalaxy,skipstandalone,skipshared,force_verify):
def cmd_verify(gamedir, skipextras, skipids, check_md5, check_filesize, check_zips, delete_on_fail, clean_on_fail, ids, os_list, lang_list, skipgalaxy,skipstandalone,skipshared, skipfiles, force_verify):
"""Verifies all game files match manifest with any available md5 & file size info
"""
item_count = 0
Expand All @@ -1955,6 +1974,7 @@ def cmd_verify(gamedir, skipextras, skipids, check_md5, check_filesize, check_z
del_file_cnt = 0
clean_file_cnt = 0
prev_verified_cnt = 0
skip_cnt = 0

items = load_manifest()

Expand Down Expand Up @@ -1983,6 +2003,10 @@ def cmd_verify(gamedir, skipextras, skipids, check_md5, check_filesize, check_z
info('verifying all known files in the manifest')
games_to_check = games_to_check_base

if skipfiles:
formattedSkipFiles = "'" + "', '".join(skipfiles) + "'"
info("skipping files that match: {%s}" % formattedSkipFiles)

handle_game_renames(gamedir,items,False)


Expand Down Expand Up @@ -2065,9 +2089,15 @@ def cmd_verify(gamedir, skipextras, skipids, check_md5, check_filesize, check_z
if itm.name is None:
warn('no known filename for "%s (%s)"' % (game.title, itm.desc))
continue

item_count += 1

skipfile_skip = check_skip_file(itm.name, skipfiles)
if skipfile_skip:
info('skipping %s (matches %s)' % (itm.name, skipfile_skip))
skip_cnt += 1
continue

itm_dirpath = os.path.join(game.title, itm.name)
itm_file = os.path.join(gamedir, game.title, itm.name)

Expand Down Expand Up @@ -2137,6 +2167,7 @@ def cmd_verify(gamedir, skipextras, skipids, check_md5, check_filesize, check_z
if not force_verify:
info('pre-verified items.. %d' % prev_verified_cnt)
info('have items.......... %d' % (item_count - missing_cnt - del_file_cnt - clean_file_cnt))
info('skipped items....... %d' % skip_cnt)
info('missing items....... %d' % (missing_cnt + del_file_cnt + clean_file_cnt))
if check_md5:
info('md5 mismatches...... %d' % bad_md5_cnt)
Expand Down Expand Up @@ -2279,7 +2310,7 @@ def main(args):
if args.wait > 0.0:
info('sleeping for %.2fhr...' % args.wait)
time.sleep(args.wait * 60 * 60)
cmd_download(args.savedir, args.skipextras, args.skipids, args.dryrun, args.ids,args.os,args.lang,args.skipgalaxy,args.skipstandalone,args.skipshared)
cmd_download(args.savedir, args.skipextras, args.skipids, args.dryrun, args.ids,args.os,args.lang,args.skipgalaxy,args.skipstandalone,args.skipshared, args.skipfiles)
elif args.command == 'import':
#Hardcode these as false since extras currently do not have MD5s as such skipgames would give nothing and skipextras would change nothing. The logic path and arguments are present in case this changes, though commented out in the case of arguments)
args.skipgames = False
Expand Down Expand Up @@ -2319,7 +2350,7 @@ def main(args):
check_md5 = not args.skipmd5
check_filesize = not args.skipsize
check_zips = not args.skipzip
cmd_verify(args.gamedir, args.skipextras,args.skipids,check_md5, check_filesize, check_zips, args.delete, args.clean,args.ids, args.os, args.lang,args.skipgalaxy,args.skipstandalone,args.skipshared, args.forceverify)
cmd_verify(args.gamedir, args.skipextras,args.skipids,check_md5, check_filesize, check_zips, args.delete, args.clean,args.ids, args.os, args.lang,args.skipgalaxy,args.skipstandalone,args.skipshared, args.skipfiles, args.forceverify)
elif args.command == 'backup':
if not args.os:
if args.skipos:
Expand Down

0 comments on commit 78e8601

Please sign in to comment.