diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..272cbd7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,38 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + - OS: [e.g. iOS] + - Browser [e.g. chrome, safari] + - Version [e.g. 22] + +**Smartphone (please complete the following information):** + - Device: [e.g. iPhone6] + - OS: [e.g. iOS8.1] + - Browser [e.g. stock browser, safari] + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..24473de --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. \ No newline at end of file diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml new file mode 100644 index 0000000..a3cf0fb --- /dev/null +++ b/.github/workflows/unit-test.yml @@ -0,0 +1,15 @@ +name: unit-test + +on: + push: + branches: [ main, dev ] + +jobs: + unit-test-tgt: + name: Unit Test + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Install dependencies + run: | + ls diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..0b6b557 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,13 @@ +[MASTER] +ignore=testbed.py + +[MESSAGES CONTROL] +disable= + invalid-name, + protected-access, + missing-docstring, + missing-function-docstring, + missing-class-docstring, + missing-module-docstring, + blacklisted-name, + import-outside-toplevel \ No newline at end of file diff --git a/.yapfignore b/.yapfignore new file mode 100644 index 0000000..93f5256 --- /dev/null +++ b/.yapfignore @@ -0,0 +1 @@ +__init__.py \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..94a2598 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +maksyuki@126.com. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..444abb5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1 @@ +## CONTRIBUTE \ No newline at end of file diff --git a/README.md b/README.md index f2e7110..8b73a67 100644 --- a/README.md +++ b/README.md @@ -1 +1,36 @@ -# ci \ No newline at end of file +

+

TreeCore CICD: A CI/CD Environment for Processor Simulation and Verification

+

+

+ + + + + + + + stars + + + + + + + + + + +

+ +## Overview +## Motivation +## Feature +## Usage + +## License +All of the TreeCore CICD codes are release under the [GPL-3.0 License](LICENSE). + +## Acknowledgement +1. [xxxxx](https://github.com/xxxx) + +## Reference diff --git a/format_code.sh b/format_code.sh new file mode 100644 index 0000000..eb031da --- /dev/null +++ b/format_code.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +yapf -ri . \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..c657024 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,8 @@ +# lint +pylint=2.12.2 +yapf=0.31.0 +mypy=0.910 + +# test +pytest=6.2.5 +pytest-cov=3.0.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f6ac63d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +toml=0.10.2 +schedule=1.2.0 \ No newline at end of file diff --git a/run_test.sh b/run_test.sh new file mode 100644 index 0000000..ff0ae1e --- /dev/null +++ b/run_test.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +pytest -rp -q \ No newline at end of file diff --git a/src/add_soc.py b/src/add_soc.py new file mode 100644 index 0000000..2309ef8 --- /dev/null +++ b/src/add_soc.py @@ -0,0 +1,94 @@ +#!/bin/python + +import os +import re +from typing import List +import cicd_config +from data_type import CoreInfo + + +class Cores(object): + def __init__(self): + self.submit_list = [] + self.core_list = [] + + def clear(self): + self.submit_list.clear() + self.core_list.clear() + + # 1. pattern: ysyx_([0-9]{6}) + # 2. in id list + def check_valid(self, val: str) -> str: + if re.match('ysyx_[0-9]{8}', val) is not None: + return val + else: + return '' + + def fill_data(self, term: List[str]): + self.submit_list.append(CoreInfo(term[0], term[1])) + + def handle_err(self, val: str): + # NOTE: need to write to the submit info + print(f'ID: error format, the err val: {val}') + + def add(self): + with open(cicd_config.SUBMIT_LIST_PATH, 'r+', encoding='utf-8') as fp: + for v in fp.readlines(): + tmp = v.split() + # print(tmp[1]) + if self.check_valid(tmp[1]) != '': + self.fill_data(tmp) + else: + self.handle_err(tmp) + + # update the core list + def update(self): + os.chdir(cicd_config.SUBMIT_DIR) + # os.system('git checkout ' + cicd_config.CUR_BRAN) + print(f'git checkout {cicd_config.CUR_BRAN}') + with open(cicd_config.CORE_LIST_PATH, 'r+', encoding='utf-8') as fp: + for v in fp.readlines(): + tmp = v.split() + # print('id: ' + val) + # filter err and spaces + if self.check_valid(tmp[0]) != '': + self.core_list.append(CoreInfo('', tmp[0])) + # print('id: ' + v.rstrip('\n')) + + self.core_list.sort(key=lambda v: v.sid) + self.submit_list.sort(key=lambda v: v.sid) + + new_id = [] + for va in self.submit_list: + is_find = False + for vb in self.core_list: + if va.sid == vb.sid: + is_find = True + break + + if is_find is False: + os.system(f'git clone {va.url} submit/{va.sid}') + print(f'git clone {va.url} submit/{va.sid}') + new_id.append(CoreInfo('', va.sid, 'F')) + + print(f'new core num: {len(new_id)}') + self.core_list += new_id + self.core_list.sort(key=lambda v: v.sid) + # print(self.core_list) + with open(cicd_config.CORE_LIST_PATH, 'w+', encoding='utf-8') as fp: + for v in self.core_list: + fp.write(v.sid + ' ' + v.flag + '\n') + + +cores = Cores() + + +def main(): + print('[add soc]') + cores.clear() + cores.add() + cores.update() + + +if __name__ == '__main__': + main() diff --git a/src/cicd_config.py b/src/cicd_config.py new file mode 100644 index 0000000..b5771de --- /dev/null +++ b/src/cicd_config.py @@ -0,0 +1,47 @@ +#!/bin/python +import os + +BRANCH_NAME_DEV = 'master' + +GMT_FORMAT = '%a %b %d %H:%M:%S %Y %z' +STD_FOMRAT = '%Y-%m-%d %H:%M:%S' + +# CUR_BRAN = '202302' +CUR_BRAN = 'main' # NOTE: just for test +HOME_DIR = os.getcwd() + '/' +DATA_DIR = HOME_DIR + '../data/' + CUR_BRAN +SUBMIT_LIST_PATH = DATA_DIR + '/submit_list' +CORE_LIST_PATH = DATA_DIR + '/core_list' +QUEUE_LIST_PATH = DATA_DIR + '/queue_list' + +DC_HOME_DIR = HOME_DIR + '../lib/dc/bes_data/syn/scr' +DC_LOG_DIR = DC_HOME_DIR + '../log' +DC_RPT_DIR = DC_HOME_DIR + '../rpt' +# NOTE: need to modify the SUBMIT_DIR path for the CICD repo +# now just for test +SUBMIT_DIR = HOME_DIR + '../tests/intg' +SUB_DIR = SUBMIT_DIR + '/submit' +RPT_DIR = SUBMIT_DIR + '/report' + +VCS_DIR = HOME_DIR + '../vcs' +VCS_RUN_DIR = VCS_DIR + '/run' +VCS_CPU_DIR = VCS_DIR + '/cpu' +VCS_SCRIPT_DIR = VCS_DIR + '/script' + + +def exec_cmd(cmd: str) -> str: + try: + ret = os.popen(cmd).read() + except Exception as e: + print(f"Error '{0}' occured when exec_cmd".format(e)) + ret = '' + return ret + + +def git_commit(path, info, push=False): + os.chdir(SUBMIT_DIR) + os.system('git add ' + path) + os.system('git commit -m "' + info + '"') + if push: + os.system('git push') + os.chdir(HOME_DIR) diff --git a/src/config_parse.py b/src/config_parse.py new file mode 100644 index 0000000..15a995f --- /dev/null +++ b/src/config_parse.py @@ -0,0 +1,90 @@ +import os +from typing import Any, Dict, Tuple +import tomli +import cicd_config +from data_type import IvConfig, VerConfig, VcsConfig, DcConfig + + +class Config(object): + def __init__(self): + self.commit_info = '' + self.iv = IvConfig() + self.ver = VerConfig() + self.vcs = VcsConfig(False, 'all', 100) + self.dc = DcConfig(100) + + def clear(self): + pass + + def check_config(self, sid) -> Tuple[bool, str]: + core_dir = cicd_config.SUB_DIR + '/' + sid + core_config_file = core_dir + '/def_config.toml' + # core_config_file = 'def_config.toml' + if os.path.isfile(core_config_file): + with open(core_config_file, 'rb') as fp: + res = tomli.load(fp) + print(res) + + cmd = f'git log origin/{cicd_config.BRANCH_NAME_DEV}' + cmd += ' --pretty=format:"%s" -1' + # print(cmd) + os.chdir(core_dir) + self.commit_info = cicd_config.exec_cmd(cmd) + print(self.commit_info) + os.chdir(cicd_config.HOME_DIR) + # self.commit_info = 'vcs' + # print(res.keys()) + std_config_keys = ['iv_config', 'ver_config', 'vcs_config'] + is_valid = False + for v in std_config_keys: + if v in res.keys( + ) and res[v]['commit_info'] == self.commit_info: + print(f'[read {v}]') + is_valid = True + self.config_parse(res[v]) + return (is_valid, self.commit_info) + else: + return (False, '') + + def iv_config_parse(self, config: Dict[str, Any]): + print(config) + + def ver_config_parse(self, config: Dict[str, Any]): + print(config) + + def vcs_config_parse(self, config: Dict[str, Any]): + # print(config) + self.vcs.wave = config['wave'] == 'on' + self.vcs.prog = config['prog'] + self.vcs.freq = config['freq'] + print(self.vcs) + + def dc_config_parse(self, config: Dict[str, Any]): + print(config) + self.dc.freq = config['freq'] + + def config_parse(self, config: Dict[str, Any]): + if self.commit_info == 'iv': + self.iv_config_parse(config) + elif self.commit_info == 'ver': + self.ver_config_parse(config) + elif self.commit_info == 'vcs': + self.vcs_config_parse(config) + elif self.commit_info == 'dc': + self.dc_config_parse(config) + + +def_config = Config() + + +def main(sid: str) -> Tuple[bool, str]: + res = def_config.check_config(sid) + if res[0]: + print(res) + else: + print('def_config.toml is not found or commit info is err!') + return res + + +if __name__ == '__main__': + main('') diff --git a/src/data_type.py b/src/data_type.py new file mode 100644 index 0000000..fd5af17 --- /dev/null +++ b/src/data_type.py @@ -0,0 +1,39 @@ +#!/bin/python + + +class CoreInfo(object): + def __init__(self, url: str, sid: str, flag='E'): + self.url = url + self.sid = sid + self.flag = flag + + +class QueueInfo(object): + def __init__(self, sid: str, date: str): + self.sid = sid + self.date = date + + +class IvConfig(object): + def __init__(self): + pass + + +class VerConfig(object): + def __init__(self): + pass + + +class VcsConfig(object): + def __init__(self, wave: bool, prog: str, freq: int): + self.wave = wave + self.prog = prog + self.freq = freq + + def __str__(self) -> str: + return f'wave: {self.wave} prog: {self.prog} freq: {self.freq}' + + +class DcConfig(object): + def __init__(self, freq: int): + self.freq = freq diff --git a/src/dc_test.py b/src/dc_test.py new file mode 100644 index 0000000..271b76b --- /dev/null +++ b/src/dc_test.py @@ -0,0 +1,249 @@ +#!/bin/python +import os +import re +# import argparse +import time +# import datetime +import cicd_config + +freqlist = ['100'] +for freq in freqlist: + with open(cicd_config.DC_HOME_DIR + '/asic_top/ysyx_229998.tcl', + encoding='utf-8') as fp: + message = '' + for line in fp: + line = re.sub('u0_rcg/u0_pll/CLK_OUT (\d+)', + 'u0_rcg/u0_pll/CLK_OUT {freq_p}'.format(freq_p=freq), + line) + message += line + with open(cicd_config.DC_HOME_DIR + '/asic_top/ysyx_229998.tcl', + 'w', + encoding='utf-8') as fp: + fp.write(message) + + vtlist = ['SVT40+LVT40'] + # user0 = time.strftime('%Y_%m_%d_%H:%M:%S') + date = time.strftime('%Y_%m_%d_%H:%M:%S') + user0 = 'CICD_YSYX_' + proc = 'SIMC(110nm)' + track = '8T' + s = '_' + f = 'M' + top_name = 'ysyx_229998' + for vt in vtlist: + corner_list = ['MAX'] + for corner in corner_list: + user = user0 + freq + f + s + vt + s + track + corner + os.system( + 'run_syn -d ysyx_229998 -c {corner_p} -t {track_p} -v {vt_p} -u {user_p} -n smic110' + .format(corner_p=corner, track_p=track, vt_p=vt, user_p=user)) + # os.system( + # './report_filter -d ysyx_229998 -u {user_p}'.format(user_p=user)) + os.chdir(cicd_config.DC_LOG_DIR) + print(f'Current working directory: {0}'.format(os.getcwd())) + + user_name = top_name + '_' + user + user_name_txt = user_name + '.txt' + result_name = '../out/dc_report_' + user_name_txt + error_log = '../out/error_log_' + user_name_txt + error_log2 = '../out/error_log2_' + user_name_txt + warning_log = '../out/warning_log_' + user_name_txt + warning_log2 = '../out/warning_log2_' + user_name_txt + + area_rpt = f'../rpt/{0}/{1}'.format(user_name, + top_name) + '.area.rpt' + area_rpt2 = f'../out/{0}'.format(user_name) + '2.area.rpt' + timing_rpt = f'../rpt/{0}/{1}'.format(user_name, + top_name) + '.timing.rpt' + timing_rpt2 = f'../out/{0}'.format(user_name) + '2.timing.rpt' + final_rpt = '../out/' + user_name_txt + log_name = user_name + '.log' + + with open(result_name, 'w', encoding='utf-8') as fp: + text = ' SYNTHESIS REPORT\n\n==============Information==============\n' + text += f'commit date: {0}\ntop_name: {1}\nfoundry: {2}\n'.format( + date, top_name, proc) + text += f'corner: {0}\ntrack: {1}\nvoltage channel: {2}\n'.format( + corner, track, vt) + text += '\n==============ERROR & WARNING==============\nErrors\n' + fp.write(text) + + # f_error = open(error_log, 'w') + # # os.remove(error_log) + i = 1 + print(log_name) + with open(log_name, 'r', encoding='utf-8') as fp: + data = fp.readlines() + for line in data: + # print(line) + pattern = re.compile(r'^Error:.*') + string = str(line) + url = re.findall(pattern, string) + with open(error_log, 'a+', encoding='utf-8') as fp2: + for urls in url: + fp2.write(urls + '\n') + + with open(error_log, 'r', encoding='utf-8') as fp: + message = '' + for line in fp: + line = re.sub( + "Error: Value for list 'object_list' must have 1 elements.", + "DEFAULT:", line) + line = re.sub( + "Error: Can't find lib_cells matching (.*)'.", + "DEFAULT:", line) + line = re.sub( + "Error: Could not open (.*).svf for writing.", + "DEFAULT:", line) + line = re.sub( + "Error: Unable to open DDC file (.*) for writing. (DDC-1)", + "DEFAULT:", line) + line = re.sub("Error: Write command failed.", "DEFAULT:", + line) + message += line + with open(error_log2, 'w+', encoding='utf-8') as fp: + fp.write(message) + + with open(error_log2, 'r', encoding='utf-8') as fp: + data = fp.readlines() + for line in data: + pattern = re.compile(r'^Error:.*') + string = str(line) + url = re.findall(pattern, string) + with open(result_name, 'a+', encoding='utf-8') as fp2: + for urls in url: + fp2.write(f'{0}'.format(i) + '. ' + urls + '\n') + i += 1 + + with open(result_name, 'a', encoding='utf-8') as fp: + fp.write('\nWarnings\n') + + # f_warning = open(warning_log, 'w', encoding='utf-8') + # os.remove(warning_log) + j = 1 + + with open(log_name, 'r', encoding='utf-8') as fp: + data = fp.readlines() + + for line in data: + pattern = re.compile(r'^Warning:.*') + string = str(line) + url = re.findall(pattern, string) + with open(warning_log, 'a+', encoding='utf-8') as fp2: + for urls in url: + fp2.write(urls + '\n') + + with open(warning_log, 'r', encoding='utf-8') as fp: + message = '' + for line in fp: + line = re.sub( + "Warning: Clock group CLK_clock_others_1 has all design clocks in one group.", + "DEFAULT:", line) + line = re.sub( + "Warning: The trip points for the library named (.*) differ from those in the library named (.*).", + "DEFAULT: (TIM-164)", line) + line = re.sub( + "Warning: The specified replacement character \(_\) is conflicting with the specified allowed or restricted character.", + "DEFAULT:", line) + message += line + with open(warning_log2, 'w+', encoding='utf-8') as fp: + fp.write(message) + + with open(warning_log2, 'r', encoding='utf-8') as fp: + data = fp.readlines() + + for line in data: + pattern = re.compile(r'^Warning:.*') + string = str(line) + url = re.findall(pattern, string) + with open(result_name, 'a+', encoding='utf-8') as fp2: + for urls in url: + fp2.write(f'{0}'.format(j) + '. ' + urls + '\n') + j = j + 1 + + with open(result_name, 'a', encoding='utf-8') as fp: + tmp = '\n\n****** Message Summary: ' + tmp += f'{0} Error(s), {1} Warning(s) ******'.format( + i - 1, j - 1) + fp.write(tmp) + + os.chdir(cicd_config.DC_RPT_DIR) + print(f'Current working directory: {0}'.format(os.getcwd())) + + rpt_name = user_name + '/' + top_name + '.statistics.rpt' + with open(result_name, 'a', encoding='utf-8') as fp: + text = '\n' + fp.write(text) + + with open(rpt_name, 'r', encoding='utf-8') as fp: + data = fp.readlines() + + timing_flag = False + timing_pass = True + for line in data: + pattern = re.compile(r'.*') + string = str(line) + url = re.findall(pattern, string) + with open(result_name, 'a+', encoding='utf-8') as fp: + for urls in url: + fp.write(urls + '\n') + + if 'Timing' in line: + timing_flag = True + + if timing_flag and re.match('CLK', line) is not None: + timing_info = line.split() + if float(timing_info[3]) < 0 or float(timing_info[4]) < 0: + timing_pass = False + + with open(result_name, 'a+', encoding='utf-8') as fp: + if timing_pass: + fp.write('\n\nf=100Mhz PASS!!!') + else: + fp.write('\n\nf=100Mhz FAIL!!!') + + os.chdir(cicd_config.DC_HOME_DIR) + + with open(result_name, 'a', encoding='utf-8') as fp: + txt = '\n\n================AREA REPORT================\n\n' + fp.write(txt) + + with open(area_rpt, 'r', encoding='utf-8') as fp: + message = '' + for line in fp: + line = re.sub('Library\(s\) Used:', '', line) + line = re.sub('scc011ums_hd_lvt_ss_v1p08_125c_ccs(.*)', '', + line) + line = re.sub('S011HD1P_X32Y2D128_SS_1.08_125(.*)', '', + line) + line = re.sub('Version:(.*)', '', line) + message += line + with open(area_rpt2, 'w+', encoding='utf-8') as fp: + fp.write(message) + + with open(area_rpt2, 'r', encoding='utf-8') as fp: + data = fp.readlines() + + for line in data: + pattern = re.compile(r'.*') + string = str(line) + url = re.findall(pattern, string) + with open(result_name, 'a+', encoding='utf-8') as fp: + for urls in url: + fp.write(urls + '\n') + + with open(result_name, 'r', encoding='utf-8') as file1: + with open(final_rpt, 'w', encoding='utf-8') as file2: + for line in file1.readlines(): + if line == '\n': + line = line.strip('\n') + file2.write(line) + os.system('cp ' + final_rpt + ' ../out/dc_report') + + print(f'Current working directory: {0}'.format(os.getcwd())) + # os.remove(error_log) + # os.remove(error_log2) + # os.remove(warning_log) + # os.remove(warning_log2) + # os.remove(area_rpt2) + # os.remove(result_name) diff --git a/src/def_config.toml b/src/def_config.toml new file mode 100644 index 0000000..050cd73 --- /dev/null +++ b/src/def_config.toml @@ -0,0 +1,21 @@ +# This is the default submit config +[meta] +version = "1.0" + +# iverilog +[iv_config] +commit_info = "iv" + +# verilator +[ver_config] +commit_info = "ver" + +[vcs_config] +commit_info = "vcs" +prog = 'all' # format: [prog]-[type] prog: [hello, memtest, rtthread] type: [flash, mem, sdram] +freq = 100 # [100, 800, step: 50] +debug = {wave = "off", prog = "hello-flash"} + +[dc_config] +commit_info = "dc" +freq = 100 # [100, 800, step: 50] \ No newline at end of file diff --git a/src/dispatch.py b/src/dispatch.py new file mode 100644 index 0000000..d973818 --- /dev/null +++ b/src/dispatch.py @@ -0,0 +1,41 @@ +#!/bin/python + +import os +import cicd_config +import config_parse +import iv_test +import ver_test +import vcs_test +# import dc_test + + +def create_dir(sid: str): + core_rpt_dir = cicd_config.RPT_DIR + '/' + sid + os.system(f'mkdir -p {core_rpt_dir}') + os.system(f'echo state: under test > {core_rpt_dir}/state') + + +def main(): + # extract one file from queue_list + with open(cicd_config.QUEUE_LIST_PATH, 'r+', encoding='utf-8') as fp: + cores = fp.readlines() + if cores != []: + tmp = cores[0].split() + # print(tmp) + create_dir(tmp[0]) + res = config_parse.main(tmp[0]) + if res[1] == 'iv': + iv_test.main() + elif res[1] == 'ver': + ver_test.main() + elif res[1] == 'vcs': + vcs_test.main() + elif res[1] == 'dc': + # dc_test.main() + pass + else: + print('this is not core in queue') + + +if __name__ == '__main__': + main() diff --git a/src/iv_test.py b/src/iv_test.py new file mode 100644 index 0000000..0b5cda1 --- /dev/null +++ b/src/iv_test.py @@ -0,0 +1,9 @@ +#!/bin/python + + +def main(): + pass + + +if __name__ == '__main__': + main() diff --git a/src/repo_update.py b/src/repo_update.py new file mode 100644 index 0000000..bc8406a --- /dev/null +++ b/src/repo_update.py @@ -0,0 +1,140 @@ +#!/bin/python +import os +from datetime import datetime +from typing import Tuple +import cicd_config +from data_type import CoreInfo, QueueInfo + + +class CoreQueue(object): + def __init__(self): + self.val_list = [] + + def clear(self): + self.val_list.clear() + + def sw_branch(self, bran_name: str): + cmd = 'git symbolic-ref --short HEAD' + # check if already in this branch + cur_bran = cicd_config.exec_cmd(cmd) + if cur_bran == (bran_name + '\n'): + return + else: + # switch to branch + print('switch to branch: ' + bran_name) + cmd = f'git checkout {bran_name}' + ret = cicd_config.exec_cmd(cmd) + print(ret) + + # return: (state: Bool, submod_name: str, std_date: str) + # state: if submod repo has new commit + def check_remote_update(self, submod_name: str) -> (Tuple[bool, str]): + os.chdir(cicd_config.SUB_DIR + '/' + submod_name) + cmd = 'git rev-parse HEAD' + local_rev = cicd_config.exec_cmd(cmd) + + self.sw_branch(cicd_config.BRANCH_NAME_DEV) + cmd = 'git remote -v update' + cicd_config.exec_cmd(cmd) + + cmd = 'git rev-parse origin/HEAD' + remote_rev = cicd_config.exec_cmd(cmd) + + cmd = f'git log origin/{cicd_config.BRANCH_NAME_DEV}' + cmd += ' --pretty=format:"%s" -1' + # print(cmd) + title_rev = cicd_config.exec_cmd(cmd) + + cmd = f'git log origin/{cicd_config.BRANCH_NAME_DEV}' + cmd += ' --pretty=format:"%ad" -1' + # print(date_rev) + date_rev = cicd_config.exec_cmd(cmd) + + std_date = datetime.strptime( + date_rev, cicd_config.GMT_FORMAT).strftime(cicd_config.STD_FOMRAT) + + os.chdir(cicd_config.HOME_DIR) + print(submod_name + ':') + print(f'local is: {local_rev}'.rstrip('\n')) + print(f'remote is: {remote_rev}'.rstrip('\n')) + print(f'git info is: {title_rev}') + print(f'commit time is: {std_date}') + return (local_rev != remote_rev, std_date) + + def pull_repo(self, submod_name: str): + os.chdir(cicd_config.SUB_DIR + '/' + submod_name) + self.sw_branch(cicd_config.BRANCH_NAME_DEV) + + cmd = 'git pull --progress -v --no-rebase "origin" ' + cmd += cicd_config.BRANCH_NAME_DEV + ret = cicd_config.exec_cmd(cmd) + print(ret) + os.chdir(cicd_config.HOME_DIR) + + def check_repo(self, core_info: CoreInfo): + ret = self.check_remote_update(core_info.sid) + # ret = (True, '2022-08-18 09:05:40') + # restart is also right + if core_info.flag == 'F': + print(f'[{core_info.sid}] first! start pull...') + self.pull_repo(core_info.sid) + self.val_list.append(QueueInfo(core_info.sid, ret[1])) + elif ret[0] is True: + print(f'[{core_info.sid}] changed!! start pull...') + # self.pull_repo(core_info.sid) + self.val_list.append(QueueInfo(core_info.sid, ret[1])) + else: + print(f'[{core_info.sid}] not changed') + + # os.chdir(cicd_config.HOME_DIR) + # check if cores have been added to the cicd database + def check_id(self): + with open(cicd_config.CORE_LIST_PATH, 'r+', encoding='utf-8') as fp: + for v in fp.readlines(): + tmp = v.split() + self.check_repo(CoreInfo('', tmp[0], tmp[1])) + + def update_queue(self): + # cicd_config.git_commit(cicd_config.SUB_DIR, '[bot] update repo') + # self.val_list = [('ysyx_23050153', '2022-08-18 09:05:40'), + # ('ysyx_23050340', '2022-08-18 09:00:38'), + # ('ysyx_23050171', '2022-08-18 09:05:47')] + self.val_list.sort(key=lambda v: v.date) + with open(cicd_config.QUEUE_LIST_PATH, 'r+', encoding='utf-8') as fp: + fp_cores = fp.readlines() + # print(fp_cores) + # print(self.val_list) + # check if new-submit cores are in self.val_list + for i, va in enumerate(fp_cores): + for j, vb in enumerate(self.val_list): + # print('va: ' + va.split()[0] + ' vb: ' + vb.sid) + if va.split()[0] == vb.sid: + fp_cores[i] = self.val_list[ + j].sid + ' ' + self.val_list[j].date + '\n' + self.val_list[j].sid = '@' + + for v in self.val_list: + if v.sid != '@': + fp_cores.append(v.sid + ' ' + v.date + '\n') + + # print(fp_cores) + # print(self.val_list) + fp.seek(0) + fp.truncate(0) + fp.flush() + fp.writelines(fp_cores) + + +core_queue = CoreQueue() + + +def main(): + os.system(f'mkdir -p {cicd_config.DATA_DIR}') + print('[repo update]') + core_queue.clear() + core_queue.check_id() + core_queue.update_queue() + + +if __name__ == '__main__': + main() diff --git a/src/soc_test.py b/src/soc_test.py new file mode 100644 index 0000000..b0630f2 --- /dev/null +++ b/src/soc_test.py @@ -0,0 +1,251 @@ +#!/bin/python + +import os +from typing import List +import cicd_config + +rpt_home = '' +dut_core = '' +lint = [''] +warning = [''] +error = [''] +run_warning = [''] +flag = [0, 0, 0] + + +def comp_test(): + global lint + global warning + global error + global run_warning + run_warning.clear() + global flag + flag = [0, 0, 0] + + os.system("cd lib/vcs/run && make comp") + with open("lib/vcs/run/compile.log", 'r', encoding='utf-8') as fp: + lint.clear() + warning.clear() + error.clear() + for line in fp: + if flag[0] == 1: + if 'vga' in line: + flag[0] = 0 + lint.pop() + continue + lint.append(line) + if line == '\n': + lint.append(line) + flag[0] = 0 + if flag[1] == 1: + warning.append(line) + if line == '\n': + warning.append(line) + flag[1] = 0 + if flag[2] == 1: + error.append(line) + if line == '\n': + error.append(line) + flag[2] = 0 + elif line[0:4] == 'Lint': + flag[0] = 1 + lint.append(line) + elif line[0:7] == 'Warning': + flag[1] = 1 + warning.append(line) + elif line[0:5] == 'Error': + flag[2] = 1 + error.append(line) + + with open(rpt_home + '/vcs_report', 'a+', encoding='utf-8') as fp: + fp.writelines('\ncore: ' + dut_core + '\n') + fp.writelines( + '\n####################\n#vcs compile log\n####################\n') + fp.writelines(error + warning + lint) + if not error and not warning and not lint: + fp.writelines("\n\n\nall clear!!\n\n\n") + + return (not error) and (not lint) + + +prog_list = [('hello.flash', 'none', 'hello_test_flash'), + ('jump.mem', 'hello.mem', 'hello_test_mem'), + ('jump.sdram', 'hello.sdram', 'hello_test_sdram'), + ('memtest.flash', 'none', 'memtest_test_flash'), + ('jump.mem', 'memtest.mem', 'memtest_test_mem'), + ('jump.sdram', 'memtest.sdram', 'memtest_test_sdram'), + ('memtest.flash', 'none', 'rtthread_test_flash'), + ('jump.mem', 'rtthread.mem', 'rtthread_test_mem'), + ('jump.sdram', 'rtthread.sdram', 'rtthread_test_sdram')] +prog_ret = [False, False, False, False, False, False, False, False, False] + + +def system_test(): + err_cnt = 0 + + with open(rpt_home + '/vcs_report', 'a+', encoding='utf-8') as fp: + if not error: + fp.writelines( + '\n##################\n#vcs system test\n##################\n') + + for i, v in enumerate(prog_list): + cmd = 'cd lib/vcs/run && ' + cmd += f'ln -sf program/{0} mem_Q128_bottom.vmf '.format(v[0]) + + if v[1] != 'none': + cmd += f'&& ln -sf program/{0} init_{1}.bin.txt '.format( + v[1], v[1].split('.')[1]) + + cmd += f'&& make run test={0}'.format(v[2]) + os.system(cmd) + + with open('lib/vcs/run/run.log', 'r', encoding='utf-8') as fp: + for line in fp: + if flag[1] == 1: + run_warning.append(line) + if line == '\n': + run_warning.append(line) + flag[1] = 0 + if line[0:7] == 'Warning': + flag[1] = 1 + run_warning.append(line) + if i < 3 and 'Hello World!' in line: + prog_ret[i] = True + elif 3 <= i and i < 6 and 'ALL TESTS PASSED!!' in line: + prog_ret[i] = True + elif 6 <= i and i < 9 and 'Hello RISC-V!' in line: + prog_ret[i] = True + + fp.writelines(run_warning) + + for i, v in enumerate(prog_ret): + (prog_name, _, prog_test) = v.split('_') + cmd = f'{0} test in {1} '.format(prog_name, prog_test) + if prog_ret[i]: + fp.writelines(cmd + 'pass!!\n') + else: + fp.writelines('!!!' + cmd + 'fail!!!\n') + err_cnt += 1 + + return err_cnt == 0 + + +def create_rpt_dir() -> bool: + is_have = False + with open(cicd_config.QUEUE_LIST_PATH, 'r+', encoding='utf-8') as fp: + cores = fp.readlines() + if cores != []: + # rec's format: ['submit/ysyx_210153', '2022-08-18' '09:05:40'] + rec = cores[0].split() + global dut_core + dut_core = rec[0].split('/')[1] + + # record queue state + os.system('mkdir -p ' + cicd_config.RPT_DIR + dut_core) + os.system('echo state: under test > ' + cicd_config.RPT_DIR + + dut_core + '/state') + + # remove record from queue + fp.seek(0) + fp.truncate(0) + fp.flush() + fp.writelines(cores[1:]) + is_have = True + + # update other cores state + fp.seek(0) + cores = fp.readlines() + cnt = 1 + for v in cores: + v_path = cicd_config.RPT_DIR + v.split()[0].split('/')[1] + os.system('mkdir -p ' + v_path) + os.system('echo state: wait ' + str(cnt) + ' cores > ' + + v_path + '/state') + cnt += 1 + + cicd_config.git_commit(cicd_config.RPT_DIR, + '[bot] update soc state file', True) + + # soc integration + global rpt_home + rpt_home = cicd_config.RPT_DIR + dut_core + rpt_home += '/' + rec[1] + '...' + rec[2] + os.system('mkdir -p ' + rpt_home) + soc_intg(rec) + else: + print('this is not core in queue') + + return is_have + + +# rec's format: ['submit/ysyx_210153', '2022-08-18' '09:05:40'] +def soc_intg(rec: List[str]): + core_path = cicd_config.SUBMIT_DIR + rec[0] + '/' + rec[0].split('/')[1] + v_format = core_path + '.v' + sv_format = core_path + '.sv' + if os.path.isfile(v_format): + os.system('cp ' + v_format + ' lib/vcs/cpu') + elif os.path.isfile(sv_format): + os.system('cp ' + sv_format + ' ' + v_format) + os.system('mv ' + v_format + ' lib/vcs/cpu') + else: + print('no core file!') + + # print('cp ' + cicd_config.SUBMIT_DIR + rec[0] + '/' + rec[0] + '.v') + os.chdir('lib/vcs/script') + os.system('python autowire.py') + os.chdir('../../') + + +def clean_vcs_env(): + os.system( + 'cd lib/vcs/run && make clean && rm temp.fp getReg* novas* verdi* -rf') + os.system('find lib/vcs/cpu/* | grep -v ysyx_210000.v | xargs rm -rf') + + +def clean_dc_env(): + os.chdir(cicd_config.HOME_DIR + 'lib/dc/bes_data/syn/') + # os.system('rm -rf log/*.log') + # os.system('rm -rf out/*.txt') + os.chdir(cicd_config.HOME_DIR) + + +def run_vcs(): + is_comp_right = comp_test() + is_run_right = False + if is_comp_right: + is_run_right = system_test() + return is_comp_right and is_run_right + + +def run_main(): + clean_vcs_env() # can not move into create_rpt_dir! + if create_rpt_dir(): + is_vcs_right = run_vcs() + if is_vcs_right: + run_dc() + cicd_config.git_commit(cicd_config.RPT_DIR, '[bot] new report!', True) + + +def run_dc(): + clean_dc_env() + os.system('rm -rf lib/vcs/cpu_dc/*.v') + os.system('cp lib/vcs/cpu/' + dut_core + '.v lib/vcs/cpu_dc/') + os.chdir('lib/vcs/script') + os.system('/autowire_new.py') + os.chdir(cicd_config.HOME_DIR) + os.chdir('lib/dc/bes_data/syn/scr/') + os.system('./syn_scr_update') + os.system('cp ../out/dc_report ' + rpt_home) + clean_dc_env() + clean_vcs_env() # can not move into run_vcs! + os.chdir(cicd_config.HOME_DIR) + + +def main(): + print('[ysyx_cicd] SoC Test') + run_main() + + +if __name__ == '__main__': + main() diff --git a/src/task.py b/src/task.py new file mode 100755 index 0000000..1b8bb59 --- /dev/null +++ b/src/task.py @@ -0,0 +1,27 @@ +#!/bin/python + +import time +import schedule +import add_soc +import repo_update +import dispatch + + +# func: +# 1. check code similarity, record commit info(freq, time) -> web +# 2. verilator test +# 3. (iverilog test) +# 4. vcs test +# struct: +# toml, database +def main_task(): + add_soc.main() + repo_update.main() + dispatch.main() + + +schedule.every(1).seconds.do(main_task) + +while True: + schedule.run_pending() + time.sleep(1) diff --git a/src/vcs_test.py b/src/vcs_test.py new file mode 100644 index 0000000..f604523 --- /dev/null +++ b/src/vcs_test.py @@ -0,0 +1,114 @@ +#!/bin/python +from enum import Enum +import os +import cicd_config + + +class LogState(Enum): + start = 0 + end = 1 + + +class VcsTest(object): + def __init__(self): + self.dut = '' + self.date = '' + self.time = '' + self.lint = [] + self.warn = [] + self.err = [] + + def clear(self): + self.dut = '' + self.date = '' + self.time = '' + self.lint = [] + self.warn = [] + self.err = [] + self.clear_dir() + + def clear_dir(self): + cmd = f'cd {cicd_config.VCS_RUN_DIR} && make clean' + cmd += ' && rm temp.fp getReg* novas* verdi* -rf' + os.system(cmd) + cmd = f'find {cicd_config.VCS_CPU_DIR}/*' + cmd += ' grep -v ysyx_210000.v | xargs rm -rf' + os.system(cmd) + + def intg_soc(self): + core_path = cicd_config.SUB_DIR + '/' + self.dut + sv_format = core_path + '.sv' + v_format = core_path + '.sv' + if os.path.isfile(sv_format): + os.system(f'cp {sv_format} {cicd_config.VCS_CPU_DIR}') + elif os.path.isfile(v_format): + os.system(f'cp {v_format} {cicd_config.VCS_CPU_DIR}') + else: + print('not found core!') + + os.chdir(cicd_config.VCS_SCRIPT_DIR) + os.system('python autowire.py') + os.chdir(cicd_config.HOME_DIR) + + def comp(self): + cmd = f'cd {cicd_config.VCS_RUN_DIR} && make comp' + os.system(cmd) + + log_state = [LogState.end, LogState.end, LogState.end] + # NOTE: receive comp log + with open(cicd_config.VCS_RUN_DIR + '/compile.log', + 'r', + encoding='utf-8') as fp: + for line in fp: + if line[0:4] == 'Lint': + log_state[0] = LogState.start + elif line[0:7] == 'Warning': + log_state[1] = LogState.start + elif line[0:5] == 'Error': + log_state[2] = LogState.start + elif line == '\n': + log_state = [LogState.end, LogState.end, LogState.end] + + if log_state[0] == LogState.start: + self.lint.append(line) + elif log_state[1] == LogState.start: + self.warn.append(line) + elif log_state[2] == LogState.start: + self.err.append(line) + + def run(self): + # err_cnt = 0 + cmd = f'cd {cicd_config.VCS_RUN_DIR} && ' + cmd += 'make all_test' + os.system(cmd) + + def gen_rpt(self): + rpt_path = cicd_config.RPT_DIR + '/' + self.dut + rpt_path += f'/{self.date}-{self.time}' + os.system(f'mkdir -p {rpt_path}') + with open(rpt_path + '/vcs_report', 'a+', encoding='utf-8') as fp: + fp.writelines(f'\ncore: {self.dut}\n') + fp.writelines( + '\n################\n#vcs compile log\n################\n' + ) + + if not self.err or not self.warn or not self.lint: + fp.writelines('all clear\n\n') + else: + fp.writelines(self.err + self.warn + self.lint) + + +vcstest = VcsTest() + + +def main(): + print('[vcs test]') + vcstest.clear() + vcstest.intg_soc() + vcstest.comp() + vcstest.run() + vcstest.gen_rpt() + + +if __name__ == '__main__': + main() diff --git a/src/ver_test.py b/src/ver_test.py new file mode 100644 index 0000000..0b5cda1 --- /dev/null +++ b/src/ver_test.py @@ -0,0 +1,9 @@ +#!/bin/python + + +def main(): + pass + + +if __name__ == '__main__': + main() diff --git a/src/wave.py b/src/wave.py new file mode 100644 index 0000000..1841236 --- /dev/null +++ b/src/wave.py @@ -0,0 +1,3 @@ +#!/bin/python + + diff --git a/template/vcs_rpt b/template/vcs_rpt new file mode 100644 index 0000000..d668ce8 --- /dev/null +++ b/template/vcs_rpt @@ -0,0 +1,3 @@ +#################### +#vcs compile log +#################### \ No newline at end of file