From 473241d65cd2069bf8168e73c0fb2bb6a5ad0d85 Mon Sep 17 00:00:00 2001 From: Daniel Smith Date: Mon, 17 Apr 2023 11:12:52 +0200 Subject: Remove the Blacklist Tool This tool is unmaintained and is not operable in the current environment. Change-Id: Ic44c8826a0f2f2ac1a84c247dcf3536f1fbac5bf Reviewed-by: Dimitrios Apostolou --- scripts/coin/blacklist_tool/README.md | 44 - scripts/coin/blacklist_tool/blacklistTool.py | 1491 -------------------------- scripts/coin/blacklist_tool/platformEnums.py | 225 ---- 3 files changed, 1760 deletions(-) delete mode 100644 scripts/coin/blacklist_tool/README.md delete mode 100644 scripts/coin/blacklist_tool/blacklistTool.py delete mode 100644 scripts/coin/blacklist_tool/platformEnums.py diff --git a/scripts/coin/blacklist_tool/README.md b/scripts/coin/blacklist_tool/README.md deleted file mode 100644 index 6093504..0000000 --- a/scripts/coin/blacklist_tool/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# Easy Blacklist Maintenance Tool - -### Requirements -1. Python 3 -2. COIN database read access (testresults.qt.io) -3. Python modules: - 1. argparse - 2. influxdb - 3. prettytable - 4. PyInquirer - -### Usage -#### Example: -`python3 blacklistTool.py --qt5dir ~/qt5/ --interactive` - -#### Parameters: -[Required] `--qt5dir ` - -[Optional] `--interactive` Enables interactive mode - -[Optional] `--fastForward ` Runs queries for blacklisted tests as usual, but fast forwards -the script to the specified test name. -[Optional] `--printActivePlatforms` Print out active platforms on startup. - -#### Optional Environment variables -`INFLUX_DB_URL` The hostname where the coin database resides. Defaults to 'testresults.qt.io' -`INFLUX_DB_PORT` The port to connect to influxdb with. SSL is required. Defaults to port 443 -`INFLUX_DB_USER` The username used to login to the COIN database -`INFLUX_DB_PASSWORD` The password used to login to the COIN database - -#### Notes -- **Interactive mode:** This is the recommended mode of operation. You will be given -a chance to enter your database username and password manually, as well as edit -the query used to retrieve blacklisted tests. -- **Automatic mode:** If a testname is found but is only a partial match, f.ex. -`[tryAcquireWithTimeout:0.2s]` versus `[tryAcquireWithTimeout]`, a report of the -mismatch will be printed upon completion of the script. -**Interactive mode** you'll be asked what to do `(edit existing, replace, or delete)`. -- **All modes:** When a test is an exact match, and has had 0 failures on any platforms in -the past 60 days (default period), the test will be removed completely from the blacklist. -- **All modes:** If a test is deleted from the blacklist and no tests remain in it, the -BLACKLIST file will be deleted. -- **All modes:** If a blacklist item's failing configurations is unchanged, but the original - file contains trailing newlines, it may be rewritten to remove the newlines. diff --git a/scripts/coin/blacklist_tool/blacklistTool.py b/scripts/coin/blacklist_tool/blacklistTool.py deleted file mode 100644 index 11a89cf..0000000 --- a/scripts/coin/blacklist_tool/blacklistTool.py +++ /dev/null @@ -1,1491 +0,0 @@ -# Copyright (C) 2019 The Qt Company Ltd. -# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 - - -from __future__ import print_function, unicode_literals -import os -import sys -import argparse -import time -import atexit -import re -from prettytable import PrettyTable -from influxdb import InfluxDBClient -from influxdb import exceptions -from platformEnums import OS, COMPILER, PLATFORM -from enum import Enum -from PyInquirer import style_from_dict, Token, prompt, Separator -from pathlib import Path - - -# Setup a clear screen function for cleaning the output window. -def clear(): - """Clear the console screen using the OS built-in methods.""" - if sys.platform == "win32": - os.system('cls') - else: - os.system('clear') - - -# Set style for interactive interface -style = style_from_dict({ - Token.QuestionMark: '#E91E63 bold', - Token.Selected: '#673AB7 bold', - Token.Separator: '#e9c01e bold', - Token.Disabled: '#8D021F bold', - Token.Instruction: '', # default - Token.Answer: '#2196f3 bold', - Token.Question: '#ffff99 bold', -}) - - -class PlatformData(Enum): - """Enum to make accessing database results more human readable in code.""" - host_arch = 0 - host_compiler = 1 - host_os = 2 - host_os_version = 3 - target_arch = 4 - target_compiler = 5 - target_os = 6 - target_os_version = 7 - -INFLUX_DB_URL = "testresults.qt.io" if not os.environ.get("INFLUX_DB_URL") else os.environ.get("INFLUX_DB_URL") -INFLUX_DB_PORT = 443 if not os.environ.get("INFLUX_DB_PORT") else int(os.environ.get("INFLUX_DB_PORT")) -fastForward = False -modifiedFiles = set() -partialMatchesSkipped = list() - - -def onExit(): - """Print out a report following completion of the script.""" - - print("\n\n\nModified files during this run:") - print("\n".join(modifiedFiles)) - - print("\nBlacklist test cases that found parial matches (NOT MODIFIED):") - for item in partialMatchesSkipped: - print(f""" -Test: [{item['testname']}] - File path: - {item['blacklistPath']} - Partial match found in file: {item['matchText']} - Existing platforms for {item['matchText']}: - {item['existingBlacklist']} - Suggested new platforms: - {item['newBlacklistItems']} - Test Results dashboard: - {item['testCaseDashboardURL']} -""") - - -atexit.register(onExit) - - -clear() # Clear the screen and get ready! - - -class editHelper(): - - def displayModifiedTable(deletedLines: str, addedLines: str) -> None: - """Displays the proposed edits to a blacklist file in a table format.""" - addRemoveTable = PrettyTable( - ["Old Blacklist Entry", "New Blacklist Entry"]) - addRemoveTable.align["Old Blacklist Entry"] = "l" - addRemoveTable.align["New Blacklist Entry"] = "l" - addRemoveTable.add_row([deletedLines.strip(), addedLines.strip()]) - print(f"\n\n Updated Blacklist for [{testname[2]}]:\n{addRemoveTable}") - - def printFailingConfigs(failedPlatforms: list) -> None: - """Displays failing configurations as reported by the database - in a table format.""" - # Prepare the table for display. It's pretty, and informational too! - failingConfigs = PrettyTable(["Host Arch", "Host Compiler", "Host OS", "Host OS Version", - "Target Arch", "Target Compiler", "Target OS", - "Target OS Version"]) - for platform in failedPlatforms: - failingConfigs.add_row(platform) - - print( - f"\nFAILING CONFIGS for {os.path.normpath(os.sep.join(testname))}:\n{failingConfigs}") - - def paintHeader(testname: tuple, blacklistedTestData: dict, failedPlatforms: list = []) -> None: - """Clears the screen and prints information relating - to the current blacklist and test case""" - clear() - print(f"\nOpening {blacklistedTestData['filePath']}") - print(f"Test Case: [{testname[2]}]") - print(f"""\nTestresults dashboard for [\ -{testname[2] if testname[2].find(':') < 0 else testname[2][:testname[2].find(':') + 1]}\ -]:\n -{blacklistedTestData['dashboardURL']}""") - if blacklistedTestData['blacklistSnip']: - print("\nCurrent Blacklist entry:\n=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=") - print(blacklistedTestData['blacklistSnip']) - print("=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=\n") - else: - print(f"\nTest [{testname[2]}] not found in blacklist...\n") - if failedPlatforms: - editHelper.printFailingConfigs(failedPlatforms) - - def checkFailingPlatformSaturation(platforms: list, platformType: str = "") -> bool: - """Returns true of the count of platforms passed is more than 3/5 - of the count list of active platforms of the same OS or platform type. - Platforms passed are assumed to be of the same OS family such - as ['windows-10 msvc-2017', 'windows7-sp1'] and will check against that - type if platformType (such as 'xcb') is not passed - - Example: ubuntu-18.04 is failing, and ubuntu-18.04, rhel-7.4, rhel-7.6, and opensuse-leap - are acive. This would mean only 25% of linux type platforms are failing, so don't use the - general term 'linux' here. - """ - - activePlatformCountofThisType = 0 - - if not platformType: # check against the OS type passed. - try: - # Get the actual type of the platforms passed. - platformType = [OS(x).isOfType for x in OS if OS( - x).normalizedValue == platforms[0].split()[0]][0] - except IndexError: - pass - - for platform in activePlatforms: - try: - # Count how many active platforms of the same type there currently are - if [OS(x).isOfType for x in OS if OS(x).normalizedValue == platform.split()[0] - ][0] == platformType: - activePlatformCountofThisType += 1 - except IndexError: - pass - - else: # Check against the platformType passed. - for platform in activePlatforms: - try: - if platformType in OS.getCanBe(platform.split(" ")[0]): - activePlatformCountofThisType += 1 - except IndexError: - pass - - for platform in platforms.copy(): - if platform not in activePlatforms: - platforms.pop(platforms.index(platform)) - - if activePlatformCountofThisType and float(len(platforms) / - activePlatformCountofThisType) >= 0.6: - return True - else: - return False - - def locateBlacklist(blacklistPath: str, testnameTuple: tuple) -> dict: - """Open the BLACKLIST file and try to locate the test in question. - Returns a dict object with data relating to the test case.""" - - returnObject = { - "filePath": "", - "fileExists": False, - "startPos": None, - "endPos": None, - "blacklistSnip": "", - "partialMatch": False, - "matchText": "", - "notFound": False, - "AdditionalLinesToKeep": set(), - "dashboardURL": "" - } - - returnObject["dashboardURL"] = f"\ -https://testresults.qt.io/grafana/d/000000009/coin-single\ --test-details?orgId=1&var-project={testnameTuple[0]}&var-\ -testcase={testnameTuple[1]}&var-testfunction\ -={testnameTuple[2] if testnameTuple[2].find(':') < 0 else testnameTuple[2][:testnameTuple[2].find(':') + 1] + ']'}\ -&var-branch=dev&var-inter=24h&from=now-60d&to=now" - - # Our initial path begins with 'qt/' and ends with the testname. Drop that and add BLACKLIST - # Raw blacklist path appears as "qt/[module]/tests/auto/[testname]/[testCase]" - path = Path(args.qt5dir, os.sep.join( - Path(blacklistPath).parts[1:-1]), "BLACKLIST") - # Some tests are run from a "/test/" subdirectory, but the blacklist is in the main - # directory, one level up. - if not path.exists() and f"{os.sep}test{os.sep}" in str(path): - path = Path(os.sep.join(path.parts[:-2]), "BLACKLIST") - # Clean up the path in case there are any non-uniform path separators. - path = os.path.normpath(path) - print(f"opening {path}\n") - print(f"Searching for test: [{testnameTuple[2]}]...") - if not os.path.exists(path): - returnObject["filePath"] = path - returnObject["fileExists"] = False - print("BLACKLIST File does not exist...") - return returnObject - - returnObject["filePath"] = path - returnObject["fileExists"] = True - - with open(path, mode="r", newline='') as blacklist: - blacklistRaw = blacklist.read() - # Locate the test name and get the bounds up until the next test item. - testnameLoc = blacklistRaw.find(f"[{testnameTuple[2]}]") - if testnameLoc < 0: - print(f"Test [{testnameTuple[2]}] not found in blacklist...\n") - returnObject["notFound"] = True - # Try to find a match with a sub-test and save it for manual review. - testnameLoc = blacklistRaw.find(f"[{testnameTuple[2]}:") - if testnameLoc >= 0: - endOfLine = blacklistRaw.find("\n", testnameLoc) - returnObject["partialMatch"] = True - returnObject["startPos"] = testnameLoc - returnObject["matchText"] = blacklistRaw[testnameLoc: endOfLine] - print( - f"Partial test match in blacklist: \ -'{blacklistRaw[testnameLoc: endOfLine]}'") - else: - returnObject["notFound"] = True - else: - returnObject["startPos"] = testnameLoc - endOfLine = blacklistRaw.find("\n", testnameLoc) - returnObject["matchText"] = blacklistRaw[testnameLoc: endOfLine] - - if not returnObject["notFound"] or returnObject["partialMatch"]: - # Look for the position of the next test name, or return -1 (end of file) - # if our test is the last test. - returnObject["endPos"] = blacklistRaw.find("[", endOfLine) if blacklistRaw.find( - "[", endOfLine) > 0 else len(blacklistRaw) - 1 - returnObject["blacklistSnip"] = blacklistRaw[returnObject["startPos"]: returnObject["endPos"]] - # Find comments to keep and add them to the list. - for line in returnObject["blacklistSnip"].splitlines(): - if line.strip().startswith('#'): - returnObject["AdditionalLinesToKeep"].add(line.strip()) - - return returnObject - - def generateNewBlacklist(blacklistedTestData: dict, failedPlatforms: list) -> set: - """Use the target OS and compiler versions to write up a list of properly formatted - blacklist entries.\nThis does not preserve the existing list.""" - newBlacklist = set() - - if blacklistedTestData["AdditionalLinesToKeep"]: - newBlacklist.update(blacklistedTestData["AdditionalLinesToKeep"]) - for target in failedPlatforms: - if target[PlatformData.target_os_version.value] == OS.Windows_10.name: - newBlacklist.add( - f"{OS[target[PlatformData.target_os_version.value]].normalizedValue} \ -{COMPILER[target[PlatformData.target_compiler.value]].value}") - else: - newBlacklist.add( - OS[target[PlatformData.target_os_version.value]].normalizedValue) - return sorted(newBlacklist) - - def deleteLines(blacklistedTestData: dict, preserveFile: bool, dryRun: bool) -> str: - """Delete the old blacklist entry from the file. If the deleted - entry was the only entry in the file and deleteLines was not told to keep - the BLACKLIST file, it will be deleted.\n - Set 'dryRun' if no changes should be made.\n - Returns a snip of what was cut out from the file, or what would - be if dryRun is set.""" - existingBlacklistData = "" - delete = False - if blacklistedTestData["startPos"] == 0 and blacklistedTestData["endPos"] is None: - delete = True # The given test spans the whole file. Delete it. - with open(blacklistedTestData["filePath"], mode="r+", newline='') as blacklist: - existingBlacklistData = blacklist.read() - else: # Snip out the test and rewrite the file. - with open(blacklistedTestData["filePath"], mode="r+", newline='') as blacklist: - existingBlacklistData = blacklist.read() - if not blacklistedTestData["startPos"] == 0: - beforeTestText = existingBlacklistData[0: - blacklistedTestData["startPos"]] - else: - beforeTestText = "" - if blacklistedTestData["endPos"] and not (blacklistedTestData["endPos"] >= - len(existingBlacklistData)): - afterTestText = existingBlacklistData[blacklistedTestData["endPos"]:] - else: - afterTestText = "" - - if len(beforeTestText + afterTestText) < 3: - beforeTestText = "" - afterTestText = "" - delete = True - if not dryRun: - blacklist.seek(0) # Reset position in file. - blacklist.write(beforeTestText + afterTestText) - blacklist.truncate() - - if delete and not preserveFile and not dryRun: - print("Deleted blacklist with 0 entries...") - # Delete the empty blacklist file. We'll create a new one if another test needs adding. - os.remove(blacklistedTestData["filePath"]) - - # Return the snippet of what's being deleted. - return existingBlacklistData[blacklistedTestData['startPos']: blacklistedTestData['endPos']] - - def writeNewEntry(blacklistedTestData: dict, linesToAdd: list, linesToDelete: str) -> dict: - """Delete the old entry (if applicable) and write the new one.\n - Returns a dict of the added and deleted snippets.""" - addedLinesSet = set(linesToAdd) - deletedLinesSet = set() - if linesToDelete: - deletedLinesSet.update(linesToDelete.splitlines()[1:]) - - # Don't rewrite the file if the lines to write are the same as the existing lines - # regardless of ordering. - if addedLinesSet.symmetric_difference(deletedLinesSet): - deletedLines = "" - if linesToDelete: - deletedLines = editHelper.deleteLines( - blacklistedTestData, True, False) - with open(blacklistedTestData["filePath"], mode="r+", newline='') as blacklist: - blacklistRaw = blacklist.read() - blacklist.seek(0) - if blacklistedTestData['startPos'] is None: - startPos = len(blacklistRaw) - else: - startPos = blacklistedTestData['startPos'] - linesToWrite = '' + f'[{testname[2]}]\n' + \ - '\n'.join(linesToAdd) + '\n' - blacklist.write( - blacklistRaw[:startPos] + linesToWrite + blacklistRaw[startPos:]) - blacklist.truncate() - # Return what's being changed. - return {"addedLines": linesToWrite, "deletedLines": deletedLines} - else: - # No change to the file was necessary - return {"addedLines": "", "deletedLines": ""} - - def determineEditRequired(existingItems: list, newItems: list) -> bool: - """Compare the list of new and old items in the blacklist - entry. Determine if there's any changes.""" - if set(existingItems).symmetric_difference(set(newItems)): - return True - else: - return False - - def getEdits(testname: tuple, blacklistedTestData: dict, failedPlatforms: list, - existingBlacklistItems: list, action: str) -> list: - """Ask the user a series of prommpts to generate a new blacklist - and provide feedback to confirm if the new list if correct.""" - success = False - while not success: - # Start the interactive editor - linesToAdd = editEntry( - testname[2], False, failedPlatforms, existingBlacklistItems, action) - # Add comment lines back in at the top. This is slightly destructive - # and may result in a comment relating to a specific platform - # appearing out of order, but it's better than dropping it. - for index, line in enumerate(blacklistedTestData['AdditionalLinesToKeep']): - linesToAdd.insert(index, line) - editHelper.displayModifiedTable( - "\n".join(existingBlacklistItems), "\n".join(linesToAdd)) - usrinput = prompt([ - { - "type": 'confirm', - 'message': "Is the new blacklist correct?", - "name": "confirm" - } - ]) - if usrinput['confirm']: - success = True - usrinput = prompt( - [{ - "type": 'confirm', - 'message': "Do you wish to perform any manual edits?", - "name": "confirm", - "default": False - }] - ) - if usrinput['confirm']: - success = False - usrinput = prompt([ - { - "type": 'editor', - 'message': "Manually edit the new entries for the blacklist.", - "name": "editor", - "default": "\n".join(linesToAdd), - "eargs": { - "editor": "default", - "ext": ".txt" - } - } - ]) - linesToAdd = usrinput['editor'].splitlines() - editHelper.displayModifiedTable( - "\n".join(existingBlacklistItems), "\n".join(linesToAdd)) - usrinput = prompt([ - { - "type": 'confirm', - 'message': "Is the new blacklist correct?", - "name": "confirm" - } - ]) - if usrinput['confirm']: - success = True - - if not success: - print("Resetting editor...") - time.sleep(1) - clear() - editHelper.paintHeader( - testname, blacklistedTestData, failedPlatforms) - return linesToAdd - - -def getActionToPerform(testname: str, blacklistedTestName: str, - hasFailures: bool, notInBlacklist: bool) -> str: - """Prompt the user for an appropriate action to take for a given test.\n - Options available change based on the context of the test in question.""" - questions = list() - # Set a bool if the found testname and the original search name are the same - isFullMatch = testname == f"[{blacklistedTestName}]" - message = "" - - if not notInBlacklist: - if isFullMatch: - message = f"Select the action to take on {testname}" - else: - message = f"""{testname} is a partial match -{f', but [{blacklistedTestName}] has 0 failures...' if not hasFailures else '.'} -What should we do?""" - questions.append( - { - 'type': 'list', - 'name': 'action', - 'message': message, - 'default': 'edit', - 'choices': [ - { - 'name': 'Edit existing', - 'value': 'edit' - } - ] - } - ) - - if hasFailures and not isFullMatch: - questions[0]['choices'].append( - { - 'name': f"Replace with [{testname[1:testname.find(':')]}]", - 'value': 'replace' - } - ) - else: - questions[0]['choices'].append( - { - 'name': 'Delete entry', - 'value': 'delete' - } - ) - - questions[0]['choices'].append( - { - 'name': f'Abort / Skip', - 'value': 'abort' - } - ) - - elif notInBlacklist and hasFailures: - questions.append( - { - 'type': 'list', - 'name': 'action', - 'message': f"{testname} is not in the existing blacklist. What should we do?", - 'default': 'abort', - 'choices': [ - { - 'name': f'Abort / Skip', - 'value': 'abort' - }, - { - 'name': f'Add [{testname}]', - 'value': 'add' - } - ] - } - ) - else: - return 'edit' - - # Abort is always available. - - answers = prompt(questions, style=style) - return (answers["action"]) - - -def editEntry(testname: str, isPartialMatch: bool, failedPlatformsRaw: list, - alreadyBlacklisted: list, action: str) -> list: - """Present the user with a series of prompts that make blacklisting and whitelisting - suggestions.\n - Failing Platform Saturation is tested against activePlatforms. If >60% of the active platforms - of a given type are failing, the general platform term will be used instead. Whitelist - suggestions will be made for the remaining acive, but passing platforms in this case.""" - - # There are a lot of lambda function here that generate or filter down lists. - # Often, the purpose is looking at the list of options to present to the user, - # but removing "Separator" objects from the list that would otherwise cause - # exceptions when examining the list data. - - # Other lambdas generate lists of related items from platformEnums.py, - # looking at various relationships between platform targets, os families - # and how a given target relates to general platform terms. - - print(f"\nEntry {action} mode for {testname}") - - allPlatforms = list() - failedPlatforms = list() - relatedPlatforms = set() - markedAsCIFlaky = False - platformCollection = dict() - whitelistPreChecked = dict() - - # Build a checkbox list for failed platforms. - # Pre-tick the options that are already in the blacklist. - if failedPlatformsRaw: - for platform in failedPlatformsRaw: - if platform[-1] == OS.Windows_10.name: - newitem = { - 'checked': True, 'name': f"{OS[platform[-1]].normalizedValue} \ -{COMPILER[platform[-3]].value}"} - else: - newitem = {'checked': True, - 'name': OS[platform[-1]].normalizedValue} - - if newitem not in failedPlatforms: - failedPlatforms.append(newitem) - - generalPlatform = OS[platform[-1]].osFamily - if generalPlatform in [PLATFORM(x).normalizedValue for x in PLATFORM]: - if generalPlatform not in platformCollection: - platformCollection[generalPlatform] = set() - platformCollection[generalPlatform].add( - f"{OS[platform[-1]].normalizedValue} {COMPILER[platform[-3]].value}" - if platform[-1] == OS.Windows_10.name else OS[platform[-1]].normalizedValue) - relatedPlatforms.add(generalPlatform) - - # Write a separator with information about general platform names to the list of options. - if alreadyBlacklisted or relatedPlatforms: - failedPlatforms.append(Separator( - '== General platform Types:==\n See https://doc.qt.io/qt-5/qguiapplication.html#\ -platformName-prop')) - - # Build the general platforms list to present in the first prompt. Avoid duplicates - # Since we're looking at platform names like 'osx', 'windows', 'rhel' - for item in alreadyBlacklisted: - # Set a flag if the existing line contains 'ci' such as "macos-10.12 ci" - # Use this flag later to pop ci back onto edited entries. - if re.search(r'\bci\b', item): - markedAsCIFlaky = True - item = item.split(" ")[0] - if item not in failedPlatforms and item in [PLATFORM(x).normalizedValue for x in PLATFORM]: - relatedPlatforms.add(item) - - # Add the general platform names if it doesn't already exist in the first half - # of the list (already blacklisted). - # If an item passes the failing platforms saturation test, pre-check it. - for item in relatedPlatforms: - if item not in [ - x for x in filter(lambda y: type(y) != - Separator, failedPlatforms) if x['name'] == item - ]: - if item == "*" and not editHelper.checkFailingPlatformSaturation( - [x['name'] for x in filter(lambda y: type(y) != - Separator, failedPlatforms) - ], "*"): - failedPlatforms.append({'checked': False, 'name': item}) - else: - failedPlatforms.append({'checked': True, 'name': item}) - - # Run the saturation test on all other targets to determine which - # general platform names we should use instead of blacklisting individual targets. - for item in platformCollection: - useGlobalPlatformTerm = editHelper.checkFailingPlatformSaturation( - list(platformCollection[item]), item) - index = None - try: - index = failedPlatforms.index([x for x in filter(lambda y: type( - y) != Separator, failedPlatforms) if x['name'] == item][0]) - except IndexError: - print(f"WARN: {item} not in list of Failed Platforms") - # Generally shouldn't happen, as any platform in platformCollection - # Should theoretically be in the failedPlatforms list. - continue - if useGlobalPlatformTerm: - failedPlatforms[index]['checked'] = True - else: - failedPlatforms[index]['checked'] = False - - # Look at each of the failed platforms and determine if - # a majority of that platform failed. If so, add the - # platform family name / type to the list of options - # and check it. - tempFailedPlatforms = [x['name'] for x in filter( - lambda y: type(y) != Separator, failedPlatforms)] - for platformType in set([OS.getType(x) for x in tempFailedPlatforms]): - # Filter the list of platforms to pass down to ones of the same type. - checked = False - index = None - try: - index = failedPlatforms.index([x for x in filter(lambda y: type( - y) != Separator, failedPlatforms) if x['name'] == item][0]) - except IndexError: - pass - - if editHelper.checkFailingPlatformSaturation( - [x for x in - filter(lambda y: OS.getType(y) == - platformType, tempFailedPlatforms) - ], platformType): - checked = True - # What platforms are in the active platforms of this type - # but have not failed? Save this for later so we can - # auto-check whitelist options. - if not whitelistPreChecked.get(platformType, []): - whitelistPreChecked[platformType] = list() - tempSet = set( - [x for x in filter(lambda y: OS.getType(y) == - platformType, activePlatforms) - ] - ).difference([ - x for x in filter(lambda y: OS.getType(y) == platformType, - tempFailedPlatforms) - ]) - if tempSet: - whitelistPreChecked[platformType].extend(list(tempSet)) - - # Check off msvc compilers in whitelist choices if windows 10 was a failed platform. - if OS.Windows_10.normalizedValue in tempFailedPlatforms and (platformType in [ - OS.Windows_10.normalizedValue, - PLATFORM.WINDOWS.normalizedValue - ]): - # Search through the raw list of failed platforms. The target compiler exists at - # index -3, and the target OS version at index -1 - failedWin10Compilers = set([COMPILER.getNormalizedValue( - x[-3]) for x in filter(lambda y: OS.Windows_10.name == y[-1], failedPlatformsRaw)]) - if not whitelistPreChecked.get(platformType, []): - whitelistPreChecked[platformType] = list() - - # Gather the list of compilers (x), and check to see which ones are currently - # active in the CI (y), but filter the active list down to only MSVC compilers (z). - # Get the difference of the sets, returning only a set of passing compilers - # which are active in the CI. - passingWin10Compilers = set([x.value for x in COMPILER if [ - x.value for y in filter(lambda z: 'msvc' in z, activePlatforms) - if x.value in y] - ] - ).difference(failedWin10Compilers) - whitelistPreChecked[platformType].extend(passingWin10Compilers) - - if index: - failedPlatforms[index]['checked'] = checked - else: - failedPlatforms.append({'name': platformType, 'checked': checked}) - - # build a checkbox list for all possible platform configs. - for key in OS: - checked = False - if key == OS.Windows_10: - for compiler in COMPILER: - if compiler.value.lower().startswith('msvc'): - allPlatforms.append( - {'checked': checked, 'name': f"{key.normalizedValue} {compiler.value}"}) - else: - allPlatforms.append( - {'checked': checked, 'name': key.normalizedValue}) - - if alreadyBlacklisted: - allPlatforms.append(Separator( - '== General platform Types:==\n See https://doc.qt.io/qt-5/qguiapplication.html#\ -platformName-prop') - ) - - for key in [PLATFORM(x).normalizedValue for x in PLATFORM]: - allPlatforms.append({'checked': False, 'name': key}) - - # Get ready to show the first prompt. - # This prompt will show platforms that are already in the blacklist, - # any new failed platforms, and general platform name suggestions. - firstAnswers = list() - - # Only present the prompt if there were any new failed platforms. - # It's possible we're in edit mode without any failures, in the case - # that the user is adding a new test or editing one that the database - # reported all-passing. - if failedPlatforms: - questions = [ - { - 'type': 'checkbox', - 'name': 'platformEdit', - 'message': '[BLACKLIST] The below have failed at least once in the past 60 days. \ -Select any to add to the blacklist. (All pre-selected by default)', - 'choices': failedPlatforms - } - ] - firstAnswers = prompt(questions, style=style) - - # Clear the list of related platforms and re-add only the ones that the user selected. - relatedPlatforms.clear() - for answer in firstAnswers['platformEdit']: - if answer in [PLATFORM(x).normalizedValue for x in PLATFORM]: - relatedPlatforms.add(answer) - - print("\n") # Visual spacer in-between prompts. - - keywordDisabledItems = list() - # Tick checkbox in allPlatforms for platforms that were selected in the first prompt. - for index, platform in enumerate(allPlatforms): - if type(platform) == Separator: - continue - - # The following block checks the selected list of platforms and disables - # specific entries that are covered by a general term such as 'osx' or - # 'xcb'. This avoids needing to manually uncheck platforms in the list - # in order to avoid redundant blacklisting. - tempOSEnumIs = None - - # The line below will return the list of "canBe" from the OS enum if - # the item exists in OS and was selected. If a value is passed in - # "platform" that isn't in OS, None will be returned. - tempOSEnum = [OS(x).canBe for x in OS if x.normalizedValue in platform['name'] - or platform['name'] in x.osFamily or platform['name'] in x.isOfType] - - # If not None or empty, check to see if the selected platform's canBe list - # contains a keyword from the list of selected related platforms. - # If it is, deselect the item so only the general platform keyword is selected. - if tempOSEnum: - tempOSEnumIs = [x for x in tempOSEnum[0] - if x in relatedPlatforms] - - if platform['name'] in firstAnswers['platformEdit'] and not tempOSEnumIs and platform['name'] != '*': - platform['checked'] = True - elif tempOSEnumIs or (platform['name'] == '*' and '*' in relatedPlatforms): - platform['disabled'] = f"Already selected for blacklisting by keyword \ -'{tempOSEnumIs[0] if tempOSEnumIs else '*'}'" - # Keep a list of indexes we mark as disabled. - keywordDisabledItems.append(index) - else: - platform['checked'] = False - - # Second prompt. Asks to add any additional platforms from the list of all possible - # blacklist options. - blacklistAnswers = list() - - # Only prompt for additional platforms if '*' was not selected. - if '*' not in relatedPlatforms: - - questions = [ - { - 'type': 'checkbox', - 'name': 'allPlatforms', - 'message': f'[BLACKLIST] Check any {"additional " if failedPlatforms else ""}\ -platforms to add.', - 'choices': allPlatforms - } - ] - - # Ask the prompt. - # Includes a quick conversion to set and back to list to strip out duplicates - blacklistAnswers = list( - set(prompt(questions, style=style)['allPlatforms'])) - - # Add disabled platform blacklist choices back into the list of blacklist answers - try: - # Find the separator in the list if there is one, start looking at - # platforms after that index. - sepIndex = allPlatforms.index( - [x for x in allPlatforms if type(x) == Separator][0]) - except ValueError or IndexError: - sepIndex = 0 - # Make a set of the disabled platform choices - tempDisabledGeneralPlatforms = set() - for item in allPlatforms[sepIndex + 1:]: - if item.get('disabled', None): - tempDisabledGeneralPlatforms.add(item['name']) - blacklistAnswers.extend(tempDisabledGeneralPlatforms) - else: - blacklistAnswers = ['*'] - - # Reset general platforms to ask about in the whitelist. - generalPlatforms = set() - - for blindex, item in enumerate(blacklistAnswers): - # Filter out unselectable separator choices from the allPlatforms - # list and search for the dict item that was selected in - # blacklistAnswers. Disable those items so they cannot be selected - # when whitelisting. - - # TODO: Is this redundant now that the whitelist options get filtered down anyway??? - - for index, platform in enumerate(allPlatforms): - if type(platform) == Separator: - continue - if platform["name"] == item: - allPlatforms[index]["disabled"] = "Already selected for blacklisting" - # Only ask for whitelisting if a blanket platform type is selected. - if item in [PLATFORM(x).normalizedValue for x in PLATFORM]: - generalPlatforms.add((item, blindex)) - - # Un-disable the items that would be blacklisted by the general platform keyword - # So they can be selected in the whitelist. - for index in keywordDisabledItems: - if allPlatforms[index]['name'] not in [x[0] for x in generalPlatforms]: - allPlatforms[index]["disabled"] = False - - # remove options from the list that are not of the same family. - for answer in blacklistAnswers.copy(): - if not PLATFORM.getIsRootType(answer) and [ - True for x in filter(lambda y: - PLATFORM.getFamily(y) == PLATFORM.getFamily( - answer), blacklistAnswers - ) - if PLATFORM.getIsRootType(x) is True - ]: - blacklistAnswers.pop(blacklistAnswers.index(answer)) - - for index, blAnswer in enumerate(blacklistAnswers): - whitelistChoices = set() - - # Get the basic list of whitelist choices based on explicitly related platforms to blAnswer - tempList = PLATFORM.getCanBe(blAnswer) - if tempList: - for choice in tempList: - # Don't add self. i.e. Don't add 'ubuntu' if 'ubuntu' is being blacklisted. - if choice not in ["*", blAnswer, f"{OS.getFamily(blAnswer)}", - f"{OS.getType(blAnswer)}"]: - whitelistChoices.add(choice) - - # Build the list of other related platforms and OSes that - # would be blacklisted by blAnswer. Duplicates are okay - # and will be stripped out later. - tempList = list() - tempList.extend(OS.getCanBe(blAnswer)) - tempList.extend(OS.getFamilyMembers(blAnswer)) - for member in OS.getTypeMembers(blAnswer): - tempList.extend([member, OS.getFamily(member)]) - - for choice in tempList: - if choice not in ["*", blAnswer, f"{OS.getFamily(blAnswer)}", - f"{OS.getType(blAnswer)}"]: - whitelistChoices.add(choice) - - # Add msvc options to the whitelist for windows 10. - if ('windows' in blAnswer and 'windows-10' in [x['name'] for x in - filter(lambda y: type(y) != Separator, - failedPlatforms) - ]) or blAnswer == '*': - for compiler in [COMPILER(x) for x in COMPILER if 'msvc' in COMPILER(x).value]: - whitelistChoices.add(compiler.value) - - whitelistChoicesFormatted = [{'name': x} for x in whitelistChoices] - - # Pre-check choices that would be blacklisted by a general term - # but have not failed recently and are active platforms in the CI. - for choice in whitelistPreChecked.get(blAnswer, []): - if PLATFORM.getIsRootType(blAnswer): - if choice in [x for x in whitelistChoices]: - whitelistChoicesFormatted[whitelistChoicesFormatted.index( - { - 'name': choice - } - )] = { - 'name': choice, 'checked': True - } - else: - if choice in [x for x in whitelistChoices]: - whitelistChoicesFormatted[whitelistChoicesFormatted.index( - { - 'name': choice - } - )] = { - 'name': choice, 'checked': True - } - - # Prompt the user for whitelist options for this blacklisted - # answer if there are any possible combinations. - if whitelistChoices: - questions = [ - { - 'type': 'checkbox', - 'name': 'exceptions', - 'message': f'[WHITELIST] Check any exceptions to \ -add to the WHITELIST for {blAnswer}.', - 'choices': sorted(whitelistChoicesFormatted, key=lambda i: i['name']) - } - ] - - whitelistAnswers = prompt(questions, style=style)['exceptions'] - - # Build up the whitelist on top of any applicable general terms provided. - # See platformEnums::OS for more information about what can be applied - # to which platform terms. - for item in whitelistAnswers: - if 'windows-10' in item and 'msvc' in item: - for compiler in [ - COMPILER(x) for x in COMPILER if x.value == item.split(' ')[1] - ]: - blacklistAnswers[index] = f"{blacklistAnswers[index]} !{compiler.value}" - else: - blacklistAnswers[index] = f"{blacklistAnswers[index]} !{item}" - - # Prompt the user for which options to mark with 'ci' if any existing - # blacklist items were marked with 'ci' - # Marking a blacklist line with 'ci' makes the line only take effect - # inside of COIN. This is useful if the test is flaky or failing explicitly - # due to a known COIN bug or infrastructure issue, but passes normally in - # a real-world environment. - if markedAsCIFlaky: - - choices = [{'name': x} for x in blacklistAnswers] - questions = [ - { - 'type': 'checkbox', - 'name': 'markForCI', - 'message': f'[CI ONLY] At least one item in the existing blacklist was marked \ -with \'ci\'. Select any new items to mark with the \'ci\' designation.', - 'choices': choices - } - ] - - flakyAnswers = prompt(questions, style=style)['markForCI'] - - for item in flakyAnswers: - blacklistAnswers[blacklistAnswers.index(item)] = f"{item} ci" - - return blacklistAnswers - - -def appendPartialMatchSkipped(testname: list, blacklistedTestData: dict, failedPlatforms: list): - partialMatchesSkipped.append( - { - "blacklistPath": blacklistedTestData["filePath"], - "testname": testname[2], - "matchText": blacklistedTestData["matchText"], - "existingBlacklist": '\n '.join(blacklistedTestData["blacklistSnip"].split("\n")), - "newBlacklistItems": '\n\ - '.join(editHelper.generateNewBlacklist(blacklistedTestData, failedPlatforms)), - "testCaseDashboardURL": blacklistedTestData['dashboardURL'] - } - ) - - -def processItem(testname: list, failedPlatforms: list): - - global fastForward # Make this global editable in this scope. - - if fastForward: - # Fast-Forward takes a test name (see arg parsing in main()) - # This skips forward in the results to the selected test if it exists. - print(f"Fast Forwarding to {args.fastForward}...") - if testname[2] == args.fastForward: - clear() - # If we found the test, cancel the fast forward. - fastForward = False - else: - return - - blacklistPath = os.sep.join(testname) - - # Find the current test in the blacklist or touch a new file. - # Return start and end bounds for the test. - blacklistedTestData = editHelper.locateBlacklist(blacklistPath, testname) - - # Abort is the default action in automatic mode. - # This skips the test if a partial match is found. - action = "abort" - haveAction = False - - if not blacklistedTestData: - # Return this iteration if there was a critical error - # such as being unable to touch a new file. - return - elif not blacklistedTestData['fileExists'] or (blacklistedTestData['notFound'] - and not blacklistedTestData['partialMatch']): - editHelper.paintHeader(testname, blacklistedTestData, failedPlatforms) - if args.interactive: - # If in interactive mode, ask the user if the test should be added. - if failedPlatforms: - action = getActionToPerform(testname[2], "", True, True) - haveAction = True - if action == 'abort': - input(f"\nEdit Aborted...\nPress Return to continue...") - clear() - return # Skip this test - elif action == 'add' and not blacklistedTestData['fileExists']: - # Touch the file since it doesn't exist and try to create it. - # This can only occur in interactive mode. - try: - # Initialize a blank file if it doesn't exist. - open( - blacklistedTestData['filePath'], mode="a", newline="") - except FileNotFoundError: - print( - f"Error writing to file at {blacklistedTestData['filePath']}... \ -Is your qt5 repository fully up-to-date?") - input( - f"Press Return to continue. Please update [{testname[2]}] in \ -{blacklistedTestData['filePath']} manually.") - clear() - else: - print( - f"\nDatabase provided no failing platforms for [{testname[2]}]. \ -Nothing to do...") - input(f"\nPress Return to continue...") - clear() - return # Skip this test - else: - # Return this iteration if the blacklist file doesn't exist or - # the test isn't in the blacklist. - return - - existingBlacklistItems = list() - - # Did we find a whole or partial match? Print the current blacklist - # and ask for action if in interactive mode. - if not blacklistedTestData["notFound"] or blacklistedTestData["partialMatch"]: - existingBlacklistItems = blacklistedTestData['blacklistSnip'].splitlines()[ - 1:] - for index, item in enumerate(existingBlacklistItems): - existingBlacklistItems[index] = existingBlacklistItems[index].strip( - ) - existingBlacklistItems = sorted(existingBlacklistItems) - - if args.interactive and blacklistedTestData["partialMatch"]: - # Get the action to perform if we found a partial match since this is a special case. - # The user may wish to update the partial match, delete it and add a new test case, - # or abort and skip the case. - if not haveAction: - editHelper.paintHeader( - testname, blacklistedTestData, failedPlatforms) - action = getActionToPerform( - blacklistedTestData["matchText"], testname[2], - True if failedPlatforms else False, False) - haveAction = True - - if (action == "edit" and blacklistedTestData["partialMatch"]) or action == "delete": - # Update the testname used for both display and editing - # if the user is editing the partial match. - testname = (testname[0], testname[1], - blacklistedTestData["matchText"][1:-1]) - - deletedLines = "" - addedLines = "" - - if failedPlatforms: - # So we have failed platforms. What should be done? - editHelper.paintHeader(testname, blacklistedTestData, failedPlatforms) - if not editHelper.determineEditRequired(existingBlacklistItems.copy(), - editHelper.generateNewBlacklist(blacklistedTestData, - failedPlatforms)): - print(f"\nBlacklist for {testname[2]} is already up-to-date.") - if args.interactive: - usrinput = prompt( - [{ - "type": 'confirm', - 'message': "Force editing?", - "name": "force", - "default": False - }] - ) - if not usrinput['force']: - return # Skip this test. - else: - return # Skip this test. - - if not haveAction: - # So the test wasn't up to date and needs editing? ask what to do. - if args.interactive: - action = getActionToPerform( - blacklistedTestData["matchText"], testname[2], True, False) - else: - if blacklistedTestData["partialMatch"]: - # Never overwrite, replace, or edit partial matches in automatic mode. - # Just log it and let the user know what was skipped. - appendPartialMatchSkipped(testname, blacklistedTestData, failedPlatforms) - return - else: - action = 'edit' - haveAction = True - - # Initialize the add/delete lists - linesToAdd = list() - linesToDelete = list() - if action in ['edit', 'replace', 'add']: - if args.interactive: - linesToAdd = editHelper.getEdits( - testname, blacklistedTestData, failedPlatforms, existingBlacklistItems, action) - else: - linesToAdd = editHelper.generateNewBlacklist( - blacklistedTestData) - elif action == "abort": - if blacklistedTestData["partialMatch"]: - # Just log the partial match and let the user know what was skipped. - appendPartialMatchSkipped(testname, blacklistedTestData, failedPlatforms) - print("\nNothing modified...") - if args.interactive: - input("Press Return to continue...") - clear() # Clear the screen after each test - return - - if (not blacklistedTestData["notFound"] - or blacklistedTestData["partialMatch"]) and action != "delete": - # Dry run the delete to see what we're deleting. It will be executed later. - linesToDelete = editHelper.deleteLines( - blacklistedTestData, True, True) - # Execute the edits. - result = editHelper.writeNewEntry( - blacklistedTestData, linesToAdd, linesToDelete) - addedLines = result['addedLines'] - deletedLines = result['deletedLines'] - - # Occurs when a user wants to edit a partial match with no failed platforms. - elif args.interactive and action == 'edit': - linesToAdd = editHelper.getEdits( - testname, blacklistedTestData, failedPlatforms, existingBlacklistItems, action) - linesToDelete = editHelper.deleteLines(blacklistedTestData, True, True) - result = editHelper.writeNewEntry( - blacklistedTestData, linesToAdd, linesToDelete) - addedLines = result.get['addedLines'] - deletedLines = result['deletedLines'] - - # Delete partial matches too, since the database doesn't track individual test cases, - # just function names and if the test function is reported as only pass, the specific test - # cases must have also passed. Don't delete it if the user selected edit in interactive mode. - elif not blacklistedTestData["notFound"] or blacklistedTestData["partialMatch"]: - action = "delete" - - if action == "delete": - print( - f"\nRemoving blacklisted test {testname[2]}\ -{'...' if failedPlatforms else ' with 0 failing configurations...'}") - deletedLines = editHelper.deleteLines( - blacklistedTestData, False, False) - - # Display a table with the changes. - if addedLines or deletedLines: - if not args.interactive: - editHelper.displayModifiedTable(deletedLines, addedLines) - - # Save the file path since we modified it. - modifiedFiles.add(blacklistedTestData["filePath"]) - - if os.path.exists(blacklistedTestData["filePath"]): - with open(blacklistedTestData["filePath"], newline='') as blacklist: - print( - f"\nNew Blacklist file:\n=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=\n\ -{blacklist.read()}=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=\n") - - else: - print("\nNothing modified...") - if args.interactive: - input("Press Return to continue...") - clear() # Clear the screen after each test - - -def getInfluxClient() -> InfluxDBClient: - client = InfluxDBClient( - host=INFLUX_DB_URL, - port=INFLUX_DB_PORT, - ssl=True, - verify_ssl=True, - username=os.environ.get("INFLUX_DB_USER") if os.environ.get( - "INFLUX_DB_USER") else "", - password=os.environ.get("INFLUX_DB_PASSWORD") if os.environ.get( - "INFLUX_DB_PASSWORD") else "", - database="coin" - ) - client._InfluxDBClient__baseurl = "{0}://{1}:{2}/{3}".format( - client._scheme, - client._host, - client._port, - "influxdb" - ) - - return client - - -def doQuery(module: str) -> dict: - """Query the database and put together a dictionary of blacklisted - tests which had at least one failure in the last 60 days.""" - client = getInfluxClient() - - def setClientProp(prop: str) -> None: - """Set the username or password with which to connect to the database""" - if prop == 'username': - os.environ["INFLUX_DB_USER"] = prompt( - [{ - "type": "input", - 'message': f"Influx DB Username:", - "name": "username", - 'default': client._username - }] - )['username'] - client._username = os.environ.get("INFLUX_DB_USER") - else: - os.environ["INFLUX_DB_PASSWORD"] = prompt( - [{ - "type": "password", - 'message': f"Influx DB Password:", - "name": "password" - }] - )['password'] - client._password = os.environ.get("INFLUX_DB_PASSWORD") - - def showDBKeysFields(message: str) -> bool: - """Print out the list of field names and tag names in the coin database - This assists with modifying the query if the user is not explicitly - familiar with the database structure.""" - if prompt( - [{ - "type": "confirm", - 'message': f"{message} Show tags/fields before editing?:", - "name": "help", - 'default': False - }] - )['help']: - clear() - newlinesep = '\n' # Workaround for not allowing '\' inside f-strings - print( - f"""TAG KEYS:\n{ - newlinesep.join([point['tagKey'] for point in - client.query('SHOW TAG KEYS from blacklisted_test').get_points()]) - }\n""") - print( - f"""FIELD KEYS:\n{ - newlinesep.join([point['fieldKey'] for point in - client.query('SHOW field KEYS from blacklisted_test').get_points()]) - }\n""") - return True - else: - return False - - if args.interactive: - # Try the pre-set user and password if they're in environment variables - if os.environ.get("INFLUX_DB_USER") and os.environ.get("INFLUX_DB_PASSWORD"): - try: - # Dummy query to check credentials. Verify read permisison. - client.query("SHOW FIELD KEYS FROM integrations") - success = True - print("Username OK...\nPassword OK...") - except exceptions.InfluxDBClientError: - print( - "Environment variable Username or password incorrect. \ -Please re-enter your credentials...") - success = False - else: - success = False - - while not success: - setClientProp('username') - setClientProp('password') - try: - # Dummy query to check credentials. Verify read permisison. - client.query("SHOW FIELD KEYS FROM integrations") - success = True - except exceptions.InfluxDBClientError as e: - print( - "Username or password incorrect. Please re-enter your credentials...") - print(e) - time.sleep(2) - clear() - - # Get the full list of blacklisted tests that had any passes at all in the last 7 days. - - selectString = "SELECT project, testCase, testFunction, id FROM blacklisted_test " - moduleString = f"and project='qt/{module}' " if module != 'qt5' else '' - whereString = f"WHERE result = 'Passed' and branch = 'dev' {moduleString}and time > now() - 7d" - - success = False - - while not success: - if args.interactive: - # Allow for editing the query. - whereString = prompt([{"type": "input", 'message': f"Edit WHERE clause:", - "name": "query", 'default': whereString}])['query'] - print("OK...") - try: - blPoints = client.query(selectString + whereString) - success = True - # The query didn't return anything in the generator. Maybe the query was bad. - if not next(blPoints.get_points(), None): - if showDBKeysFields("Query returned 0 results."): - success = False - else: - success = False - clear() - except exceptions.InfluxDBClientError as e: - showDBKeysFields( - f"\nError while running query: {e}Please modify the query and try again...\n") - success = False - else: - blPoints = client.query(selectString + whereString) - success = True - - # Generate a dictionary of the testnames, each with an empty dict. - tests = {} - for point in blPoints.get_points(): - tests[(point["project"], point["testCase"], point["testFunction"])] = {} - - # Query for all executed configurations that had at least one failure in the last 60 days. - for test in tests: - # The whitespace line below lets us use carriage return and overwrite the current line - # for each test name being processed. Getting the actual console window width is not - # lightweight or pretty, so this works fine without much risk of garbage being displayed. - print("\r \ - ", end="") - print( - f"\rProcessing blacklisted test: \ -\"{os.path.normpath(os.sep.join(test).strip())}\"", end="") - queryForFail = f"SELECT id, host_arch, host_compiler, host_os, host_os_version, \ -target_arch, target_compiler, target_os, target_os_version FROM blacklisted_test WHERE \ -project = '{test[0]}' and testCase = '{test[1]}' and testFunction = '{test[2]}' \ -and branch = 'dev' and result = 'Failed' and time> now()-60d" - - failures = client.query(queryForFail) - - failedPlatforms = {} - # Verify that the query returned at least one point (one failed configuration). - if next(failures.get_points(), None) is not None: - for point in failures.get_points(): - if test not in failedPlatforms: - # Make the test name a set object so it will be unique. - # This seems redundant at the moment because the configurations are - # addressed as tests[testname][testname] in __main__ - # Maybe it can be fixed elegantly. - failedPlatforms[test] = set() - # Add the configuration to the set object. - # If it's an exact duplicate it will be ignored. - failedPlatforms[test].add( - (point["host_arch"], point["host_compiler"], point["host_os"], - point["host_os_version"], point["target_arch"], point["target_compiler"], - point["target_os"], point["target_os_version"]) - ) - tests[test] = failedPlatforms - else: - # The query returned 0 points. - # Set the object in tests to None so we can safely check for it later. - tests[test] = None - - print("\nDone...") - if args.interactive: - # Cosmetic sleep for the UI # Everyone needs their beauty rest! - time.sleep(2) - return tests - - -def getActivePlatforms() -> list: - """Runs a query on the database to gether a list of recently - run targets. This list can be used to understand what platforms - are currently active in the CI.""" - - client = getInfluxClient() - - result = client.query( - "SELECT id, target_os, target_os_version, target_compiler FROM workitem where branch = \ -'dev' and time >= now()-30d GROUP BY target_os, target_os_version, target_compiler") - - activeTargets = set() - # Create a unique set of the recently run platforms - for point in result.get_points(): - activeTargets.add( - (point['target_os'], point['target_os_version'], point['target_compiler'])) - - friendlyTargetNames = set() - ignoredPlatforms = set() - for target in activeTargets: - try: - if target[1] == OS.Windows_10.name: - friendlyTargetNames.add( - f"{OS[target[1]].normalizedValue} {COMPILER[target[2]].value}") - else: - friendlyTargetNames.add(OS[target[1]].normalizedValue) - except KeyError: - # A platform returned by the database that recently reported data - # is "active", but we don't care about it because it's not in our - # enums for platforms that run tests and can be blacklisted. - # This is to be expected for any platform that has tests disabled - # on all configurations, or for a platform that does not report - # itself in a uniquely identifiable way to the blacklister. - ignoredPlatforms.add( - target[1] if not target[1] == OS.Windows_10.name else f'{target[1]} {target[2]}') - - if ignoredPlatforms: - printableIgnoreList = '\n '.join(sorted(ignoredPlatforms)) - print(f""" -WARN: The following platforms are not present in platformEnums.py, - but have recently run workitems in the CI. If these platforms - are not running tests, this message can be safely ignored. - Otherwise, platformEnums.py may need to be updated. - - =-=-=-=-=-=-=-=-=-=-=- - {printableIgnoreList} - =-=-=-=-=-=-=-=-=-=-=- - -""") - if args.printActivePlatforms: - prettyPlatforms = PrettyTable(["OS Type", "OS Version", "Compiler Target"]) - for platform in sorted(activeTargets): - prettyPlatforms.add_row(platform) - - print("All active platforms:") - print(prettyPlatforms) - - if args.interactive: - input("Press return to continue...") - return list(friendlyTargetNames) - - -def validateQt5Dir() -> dict: - args.qt5dir = os.path.normpath(args.qt5dir) - - if not args.qt5dir.endswith(os.sep): - args.qt5dir = args.qt5dir + os.sep - - if not os.path.exists(args.qt5dir): - return({'exists': False, 'module': None}) - else: - module = args.qt5dir.split(os.sep)[-2] - if not module == 'qt5': - # Strip off the module since the test returns it from the database - args.qt5dir = args.qt5dir[:args.qt5dir[:-1].rfind(os.sep) + 1] - return({'exists': True, 'module': module}) - - -if __name__ == "__main__": - - parser = argparse.ArgumentParser() - parser.add_argument('--interactive', '-i', action='store_true', dest="interactive", - help="Set --interactive or -i to confirm changes and edit entries.") - parser.add_argument('--qt5dir', dest='qt5dir', type=str, required=True, - help='The full path to a checked-out qt5 supermodule or a single submodule') - parser.add_argument('--fastForward', dest='fastForward', - type=str, help='Test Case name to Fast Forward to.') - parser.add_argument('--printActivePlatforms', '-p', action='store_true', dest="printActivePlatforms", - help="Print out active COIN platforms on startup") - args = parser.parse_args() - - if args.fastForward: - fastForward = True - - moduleValidation = validateQt5Dir() - if not moduleValidation['exists']: - print( - f"Path to qt5 or qt5 submodule does not exist. \ -Please verify that the path is correct: {args.qt5dir}") - exit(0) - - # Gather the list of blacklisted tests and their failed configurations from the last 60 days. - tests = doQuery(moduleValidation['module']) - - # Query the most recent integration and see which platforms are currently active in COIN. - # This is a global that gets used any time we check a given platform type's failing percentage. - # See editHelper.checkFailingPlatformSaturation() - activePlatforms = getActivePlatforms() - - for testname in tests: - clear() - # The test had no failures. See about removing it from the blacklist entirely. - if tests.get(testname) is None: - processItem(testname, None) - print("\n\n") - else: - # Update the blacklist configurations with failed platforms. - processItem(testname, tests[testname][testname]) - print("\n\n") diff --git a/scripts/coin/blacklist_tool/platformEnums.py b/scripts/coin/blacklist_tool/platformEnums.py deleted file mode 100644 index f5f3951..0000000 --- a/scripts/coin/blacklist_tool/platformEnums.py +++ /dev/null @@ -1,225 +0,0 @@ -# Copyright (C) 2019 The Qt Company Ltd. -# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 - -from enum import Enum - - -class OS(Enum): - """Defines properties of OS types. - Enumeration names are exact matches for the platform targets reported by - the database.\n - Tuple values are as follows, with explanation:\n - [1] OS name/version pair values that are read and accepted by the blacklist. - This is what is written to BLACKLIST files.\n - [2] The family os OSs the target belongs to. This is used when checking how - many of a given OS family are currently failing.\n - [3] "canBe" list. This list describes which platforms apply to a given OS target. - This is used when determining which oses should be included under platform terms - such as "xcb"\n - [4] The general platform term used to describe the OS, such as "linux", "osx", or "windows" - """ - openSUSE_15_0 = ("opensuse-leap", "suse", - ["*", "linux", "xcb", "wayland", "openwfd", "directfb", "minimal"], "linux") - openSUSE_42_3 = ("opensuse-42.3", "suse", - ["*", "linux", "xcb", "wayland", "openwfd", "directfb", "minimal"], "linux") - SLES_15 = ("sles-15.0", "suse", ["*", "linux", "xcb", - "wayland", "openwfd", "directfb", "minimal"], "linux") - SLED_15 = ("sled-15.0", "suse", - ["*", "linux", "xcb", "wayland", "openwfd", "minimal"], "linux") - Ubuntu_16_04 = ("ubuntu-16.04", "ubuntu", ["*", "linux", "ubuntu", - "xcb", "directfb", "wayland", "openwfd", "minimal"], "linux") - Ubuntu_18_04 = ("ubuntu-18.04", "ubuntu", ["*", "linux", "ubuntu", - "xcb", "directfb", "wayland", "openwfd", "minimal"], "linux") - RHEL_6_6 = ("rhel-6.6", "rhel", ["*", "linux", "rhel", "xcb", - "wayland", "directfb", "openwfd", "minimal"], "linux") - RHEL_7_4 = ("rhel-7.4", "rhel", ["*", "linux", "rhel", "xcb", - "wayland", "directfb", "openwfd", "minimal"], "linux") - RHEL_7_6 = ("rhel-7.6", "rhel", ["*", "linux", "rhel", "xcb", - "wayland", "directfb", "openwfd", "minimal"], "linux") - OSX_10_11 = ("osx-10.11", "osx", - ["*", "osx", "cocoa", "directfb", "minimal", "offscreen"], "osx") - MacOS_10_12 = ("osx-10.12", "osx", - ["*", "osx", "cocoa", "directfb", "minimal", "offscreen"], "osx") - MacOS_10_13 = ("osx-10.13", "osx", - ["*", "osx", "cocoa", "directfb", "minimal", "offscreen"], "osx") - MacOS_10_14 = ("osx-10.14", "osx", - ["*", "osx", "cocoa", "directfb", "minimal", "offscreen"], "osx") - Windows_7 = ("windows-7sp1", "windows-7sp1", - ["*", "windows", "windows-7", "kms", "minimal", "offscreen"], "windows") - Windows_10 = ("windows-10", "windows-10", - ["*", "windows", "windows-10", "kms", "minimal", "offscreen"], "windows") - WinRT_10 = ("winrt", "winrt", [ - "*", "windows", "winrt", "kms", "minimal", "offscreen"], "windows") - Android_ANY = ("android", "android", [ - "*", "linuxfb", "eglfs", "directfb", "openwfd", "minimal"], "android") - QEMU = ("b2qt", "b2qt", ["*", "linuxfb", - "eglfs", "directfb", "minimal"], "b2qt") - - def __init__(self, normalizedValue: str, osFamily: str, canBe: list, isOfType: str): - """Make the tuple values named so the can be retrieved with a - simple accessor like OS.RHEL_7_4.normalizedValue""" - self.normalizedValue = normalizedValue - self.osFamily = osFamily - self.canBe = canBe - self.isOfType = isOfType - - @classmethod - def count(cls, typeRequested: str) -> int: - count = 0 - for entry in cls: - if entry.isOfType == typeRequested or typeRequested in entry.normalizedValue: - count += 1 - - return count - - @classmethod - def getFamily(cls, normalizedValue: str) -> str: - for entry in cls: - if entry.normalizedValue == normalizedValue: - return entry.osFamily - return "" - - @classmethod - def getType(cls, normalizedValue: str) -> str: - for entry in cls: - if entry.normalizedValue == normalizedValue: - return entry.isOfType - return "" - - @classmethod - def getCanBe(cls, normalizedValue: str) -> list: - for entry in cls: - if entry.normalizedValue == normalizedValue: - return entry.canBe - return [] - - @classmethod - def getFamilyMembers(cls, familyName: str) -> list: - return [entry.normalizedValue for entry in cls if (entry.osFamily == familyName or - familyName == '*')] - - @classmethod - def getTypeMembers(cls, typeName: str) -> list: - return [entry.normalizedValue for entry in cls if (entry.isOfType == typeName or - typeName == '*')] - - -class COMPILER(Enum): - """Mainly used when determining MSVC compilers - to blacklist.""" - GCC = "gcc" - Clang = "clang" - Mingw73 = "mingw-7.3" - MSVC2015 = "msvc-2015" - MSVC2017 = "msvc-2017" - MSVC2019 = "msvc-2019" - - @classmethod - def getNormalizedValue(cls, requestName: str) -> str: - for entry in cls: - if entry.name == requestName: - return entry.value - elif entry.value == requestName: - return entry.value - return "" - - @classmethod - def isCompiler(cls, requestName: str) -> bool: - for entry in cls: - if requestName == entry.value: - return True - return False - - -class PLATFORM(Enum): - """Defines properties of PLATFORM types. - Tuple values are as follows, with explanation:\n - [1] Platform name values that are read and accepted by the blacklist. - This is what is written to BLACKLIST files.\n - [2] "canBe" list. This list describes which platforms apply to a given OS target. - This is used when determining which oses should be included under platform terms - such as "xcb"\n - [3] Describes the base OS type if the platform itself describes some version or - distribution of an OS.\n - [4] Denotes if the platform type is a base type that cannot be whitelisted, such as linux, - windows, or osx.\n - General platform names that are acceptable in blacklists can be found at - https://doc.qt.io/qt-5/qguiapplication.html#platformName-prop - \n - The canBe values show relations so the tool can blacklist - platforms with exceptions such as "xcb !ubuntu""" - - ALL = ("*", [], "", True) - ANDROID = ("android", ["eglfs", "linuxfb", "directfb", - "minimal", "offscreen", "linux", "*"], "", False) - COCOA = ("cocoa", ["osx", "directfb", "minimal", - "directfb", "offscreen", "*"], "", False) - # QSysInfo::ProductType() returns "osx" for all macOS systems, - # regardless of Apple naming convention - OSX = ("osx", ["directfb", "minimal", "directfb", - "offscreen", "*"], "osx", True) - DIRECTFB = ("directfb", ["osx", "android", "cocoa", - "qnx", "linux", "rhel", "ubuntu", "*"], "", False) - EGLFS = ("eglfs", ["android", "ios", "qnx", "windows", - "windows_10", "linux", "rhel", "ubuntu", "*"], "", False) - IOS = ("ios", ["*"], "ios", True) - KMS = ("kms", ["windows", "windows-10", "*"], "", False) - LINUXFB = ("linuxfb", ["linux", "rhel", "ubuntu", - "windows", "windows-10", "osx", "*"], "", False) - MINIMAL = ("minimal", ["linux", "rhel", "ubuntu", - "windows", "windows-10", "osx", "*"], "", False) - OFFSCREEN = ("offscreen", ["osx", "android", "cocoa", "ios", "qnx", - "windows", "windows_10", "linux", "rhel", "ubuntu", "*"], "", False) - OPENWFD = ("openwfd", ["osx", "android", "cocoa", "ios", "qnx", - "windows", "windows_10", "linux", "rhel", "ubuntu", "*"], "", False) - QNX = ("qnx", ["*"], "", True) - WINDOWS = ("windows", ["kms", "minimal", "*"], "windows", True) - WINDOWS_10 = ("windows-10", ["kms", "windows", - "minimal", "offscreen", "*"], "windows", False) - WAYLAND = ("wayland", ["linux", "rhel", "ubuntu", "*"], "", False) - XCB = ("xcb", ["linux", "rhel", "ubuntu", "*"], "", False) - LINUX = ("linux", ["*", "eglfs", "directfb", "linuxfb", - "offscreen", "minimal", "xcb"], "linux", True) - RHEL = ("rhel", ["linux", "directfb", "eglfs", "linuxfb", - "minimal", "offscreen", "openwfd", "xcb", "*"], "linux", False) - UBUNTU = ("ubuntu", ["linux", "directfb", "eglfs", "linuxfb", - "minimal", "offscreen", "openwfd", "xcb", "*"], "linux", False) - - def __init__(self, normalizedValue: str, canBe: list, osFamily: str, isRootType: bool): - self.normalizedValue = normalizedValue - self.canBe = canBe - self.osFamily = osFamily - self.isRootType = isRootType - - @classmethod - def getNormalizedValue(cls, requestName: str) -> str: - for entry in cls: - if entry.name == requestName: - return entry.normalizedValue - elif entry.normalizedValue == requestName: - return entry.normalizedValue - return "" - - @classmethod - def getCanBe(cls, normalizedValue: str) -> list: - if normalizedValue == '*': - return [x.normalizedValue for x in cls if x.normalizedValue != '*'] - else: - for entry in cls: - if entry.normalizedValue == normalizedValue: - return entry.canBe - return [] - - @classmethod - def getFamily(cls, normalizedValue: str) -> str: - for entry in cls: - if entry.normalizedValue == normalizedValue: - return entry.osFamily - return "" - - @classmethod - def getIsRootType(cls, normalizedValue: str) -> bool: - for entry in cls: - if entry.normalizedValue == normalizedValue: - return entry.isRootType - return False -- cgit v1.2.1