From 97abac154801a7d1cd54c06e7ad56b67a8624624 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Thu, 3 Oct 2024 12:00:36 +0100 Subject: [PATCH 01/16] Resolve paths when blaming files Both `git check-ignore` and `git blame` will error if run on a symbolic link, so resolve the path before running them. --- web/client/codechecker_client/blame_info.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/web/client/codechecker_client/blame_info.py b/web/client/codechecker_client/blame_info.py index ed52db27f9..32d46731f4 100644 --- a/web/client/codechecker_client/blame_info.py +++ b/web/client/codechecker_client/blame_info.py @@ -39,8 +39,9 @@ def __get_tracking_branch(repo: Repo) -> Optional[str]: def __get_blame_info(file_path: str): """ Get blame info for the given file. """ try: - repo = Repo(file_path, search_parent_directories=True) - if repo.ignored(file_path): + real_path = os.path.realpath(file_path) + repo = Repo(real_path, search_parent_directories=True) + if repo.ignored(real_path): LOG.debug("File %s is an ignored file", file_path) return None except InvalidGitRepositoryError: @@ -59,7 +60,7 @@ def __get_blame_info(file_path: str): pass try: - blame = repo.blame_incremental(repo.head.commit.hexsha, file_path) + blame = repo.blame_incremental(repo.head.commit.hexsha, real_path) res = { 'version': 'v1', From 7aa78abd62ea18b632c8f36eca9aef6757738cbd Mon Sep 17 00:00:00 2001 From: Baghirov Feyruz <113597150+feyruzb@users.noreply.github.com> Date: Fri, 29 Nov 2024 10:05:53 +0100 Subject: [PATCH 02/16] Automatic addition of database before connecting to it (#4316) * added a function that creates a database before addproduct() function connects to it * renamed function to follow convention of naming * retrigger checks * removed unnecessary test case * test commit * changed try catch block * working implementation * commit to check if test cases will work * fixed sqlite error * fixed tests failing * changed behaviour of add_test_package_product to work with new fucntionality of add_product * possible fix of psql_psycopq2 * removed changed to creation of test package product * probable fix * final working implementation * removed automatic prevention of previous jobs, as it was already implemented by other pr * comments applied * probable fix multiple server database prohibition problem * fix of failing test --- .../codechecker_server/api/product_server.py | 67 +++++++++++++++++++ web/server/codechecker_server/server.py | 32 +++++++++ web/server/vue-cli/src/views/NewFeatures.vue | 10 +-- .../products/test_config_db_share.py | 15 +++-- .../functional/products/test_products.py | 10 +-- web/tests/functional/store/test_store.py | 6 +- web/tests/libtest/codechecker.py | 5 +- 7 files changed, 123 insertions(+), 22 deletions(-) diff --git a/web/server/codechecker_server/api/product_server.py b/web/server/codechecker_server/api/product_server.py index f7cbaeadb6..f7d4358ca4 100644 --- a/web/server/codechecker_server/api/product_server.py +++ b/web/server/codechecker_server/api/product_server.py @@ -15,6 +15,9 @@ from sqlalchemy.sql.expression import and_ +from sqlalchemy import create_engine, exc +from sqlalchemy.engine.url import URL + import codechecker_api_shared from codechecker_api.ProductManagement_v6 import ttypes @@ -318,6 +321,57 @@ def getProductConfiguration(self, product_id): return prod + @timeit + def __add_product_support(self, product): + """ + Creates a database for the given product, + to assist addProduct() function that connects to + an already existing database. + """ + + product_info = product.connection + if product_info.engine == 'sqlite': + LOG.info("Using SQLite engine, skipping database creation") + return True + + db_host = product_info.host + db_engine = product_info.engine + db_port = int(product_info.port) + db_user = convert.from_b64(product_info.username_b64) + db_pass = convert.from_b64(product_info.password_b64) + db_name = product_info.database + + engine_url = URL( + drivername=db_engine, + username=db_user, + password=db_pass, + host=db_host, + port=db_port, + database='postgres' + ) + engine = create_engine(engine_url) + try: + with engine.connect() as conn: + conn.execute("commit") + LOG.info("Creating database '%s'", db_name) + conn.execute(f"CREATE DATABASE {db_name}") + conn.close() + except exc.ProgrammingError as e: + LOG.error("ProgrammingError occurred: %s", str(e)) + if "already exists" in str(e): + LOG.error("Database '%s' already exists", db_name) + return False + else: + LOG.error("Error occurred while creating database: %s", str(e)) + return False + except exc.SQLAlchemyError as e: + LOG.error("SQLAlchemyError occurred: %s", str(e)) + return False + finally: + engine.dispose() + + return True + @timeit def addProduct(self, product): """ @@ -352,6 +406,19 @@ def addProduct(self, product): codechecker_api_shared.ttypes.ErrorCode.GENERAL, msg) + # Check if the database is already in use by another product. + db_in_use = self.__server.is_database_used(product) + if db_in_use: + LOG.error("Database '%s' is already in use by another product!", + product.connection.database) + raise codechecker_api_shared.ttypes.RequestFailed( + codechecker_api_shared.ttypes.ErrorCode.DATABASE, + "Database is already in use by another product!") + + # Add database before letting product connect to it + if self.__add_product_support(product): + LOG.info("Database support added successfully.") + # Some values come encoded as Base64, decode these. displayed_name = convert.from_b64(product.displayedName_b64) \ if product.displayedName_b64 else product.endpoint diff --git a/web/server/codechecker_server/server.py b/web/server/codechecker_server/server.py index 56bdae5d02..5e6febb054 100644 --- a/web/server/codechecker_server/server.py +++ b/web/server/codechecker_server/server.py @@ -25,6 +25,7 @@ import multiprocess from sqlalchemy.orm import sessionmaker +from sqlalchemy.engine.url import make_url from sqlalchemy.sql.expression import func from thrift.protocol import TJSONProtocol from thrift.transport import TTransport @@ -753,6 +754,9 @@ def __init__(self, permissions.initialise_defaults('SYSTEM', { 'config_db_session': cfg_sess }) + + self.cfg_sess_private = cfg_sess + products = cfg_sess.query(ORMProduct).all() for product in products: self.add_product(product) @@ -896,6 +900,34 @@ def add_product(self, orm_product, init_db=False): self.__products[prod.endpoint] = prod + def is_database_used(self, conn): + """ + Returns bool whether the given database is already connected to by + the server. + """ + + # get the database name from the database connection args + conn = make_url(conn.connection) + is_sqlite = conn.engine == 'sqlite' + + # create a tuple of database that is going to be added for comparison + to_add = (f"{conn.engine}+pysqlite" if is_sqlite + else f"{conn.engine}+psycopg2", + conn.database, conn.host, conn.port) + + # dynamic_list contains the currently connected databases to servers + dynamic_list = [(make_url(a.connection).drivername, + make_url(a.connection).database, + make_url(a.connection).host, + make_url(a.connection).port) + for a in self.cfg_sess_private.query(ORMProduct).all()] + + self.cfg_sess_private.commit() + self.cfg_sess_private.close() + + # True if found, False otherwise + return to_add in dynamic_list + @property def num_products(self): """ diff --git a/web/server/vue-cli/src/views/NewFeatures.vue b/web/server/vue-cli/src/views/NewFeatures.vue index 8a98c713b5..1e1e919c87 100644 --- a/web/server/vue-cli/src/views/NewFeatures.vue +++ b/web/server/vue-cli/src/views/NewFeatures.vue @@ -2,8 +2,6 @@ - -