Bump urllib3 from 1.26.16 to 1.26.18 in /backend/src

.gitignore

@@ -0,0 +1,6 @@
+# ignore visual studio code directory
+.vscode/
+# ignore data directory incase cometa was created using cometa.sh script
+data/
+# ignore _template files
+*_template

.gitlab-ci.yml

@@ -3,6 +3,7 @@
 # for every installation / server / client
 # ######################################## #
 # Changelog:
+# 2022-01-26 RRO Added GITCLIENTID, GITCLIENTSECRET
 # 2022-01-06 RRO Added gunicorn reload for behave and django
 # ######################################## #
 
@@ -11,14 +12,14 @@
 include:
   - 'gitlab-ci-yamls/*.yml'
 
-# set GIT_STRATEGY to none to avoid 
 # pulling content accidentally and 
 # losing old changes
 variables:
 #  CI_DEBUG_TRACE: "true" # show verbose information 
-#  GIT_STRATEGY: none
+#  GIT_STRATEGY: clone
   COMETA_EMAIL_ENABLED: 'True'
   COMETA_REPLACE_FAVICON_IN: "front/src/index.html front/src/manifest.json front/src/welcome.html"
+  COMETA_TOTAL_BROWSER_VERSION: 10 # number of latest browser version that will be pulled.
 
 # set deploy order in which jobs
 # will be executed
@@ -29,27 +30,46 @@ stages:
 # redundancy.
 .deployment_script: &deployment_script
   stage: deploy
-  script:
+  script:    
     #########################################################
-    # OLD REPOSITORY CODE
+    # REPLACE
     #########################################################
-    # cd to DEPLOY_BACKEND_FOLDER before doing anything
-    # - cd $DEPLOY_BACKEND_FOLDER
-    # stash old changes before pulling new changes
-    # - sudo git stash
-    # make sure we are in correct branch
-    # - sudo git checkout $CI_COMMIT_BRANCH
-    # pull new changes
-    # - sudo git pull
+    # update docker-compose file depending on server variables
+    - sed -i 's/<outside_port>/'$DOCKER_HTTP_PORT'/g;s/<server>/'$DOCKER_OPENIDC_CONFIG_EXT'/g' docker-compose.yml
+    # Replace secrets in openidc
+    - sed -i 's/@@COMETA_CRYPTO_PASSPHRASE@@/'$COMETA_CRYPTO_PASSPHRASE'/g' front/apache-conf/openidc.conf_${DOCKER_OPENIDC_CONFIG_EXT}
+    # Replace google client information
+    - sed -i 's/@@COMETA_GOOGLE_CLIENTID@@/'$COMETA_GOOGLE_CLIENTID'/g' front/apache-conf/metadata/accounts.google.com.client
+    - sed -i 's/@@COMETA_GOOGLE_SECRETKEY@@/'$COMETA_GOOGLE_SECRETKEY'/g' front/apache-conf/metadata/accounts.google.com.client
+    # Replace git client secret for access to git.amvara.de
+    - sed -i 's/@@GITCLIENTID@@/'$GITCLIENTID'/g' front/apache-conf/metadata/git.amvara.de.client
+    - sed -i 's/@@GITCLIENTSECRET@@/'$GITCLIENTSECRET'/g' front/apache-conf/metadata/git.amvara.de.client
+    # Replace Stripe client keys
+    - sed -i 's/@@COMETA_STRIPE_PUBLIC_LIVE_KEY@@/'$COMETA_STRIPE_PUBLIC_LIVE_KEY/'g' front/src/app/deploy-tokens.ts
+    - sed -i 's/@@COMETA_STRIPE_PUBLIC_TEST_KEY@@/'$COMETA_STRIPE_PUBLIC_TEST_KEY'/g' front/src/app/deploy-tokens.ts
+    # Replace co.meta favicon depending on the branch
+    - for FILE in ${COMETA_REPLACE_FAVICON_IN}; do sed -i 's/@@BRANCH@@/'$CI_COMMIT_BRANCH'/g' $FILE; done
+
+
     #########################################################
-
+    # COPY CODE TO DEPLOY_BACKEND_FOLDER
+    #########################################################
+    # make sure the deployment folder exists
+    - mkdir -p $DEPLOY_BACKEND_FOLDER
+    # copy all the data from current directory to the deployment directory
+    - rsync -av --exclude={".git/","*.so"} ./ $DEPLOY_BACKEND_FOLDER/.
+    # change directory to deployment folder
+    - cd $DEPLOY_BACKEND_FOLDER
+
     #########################################################
     # BACKEND
     #########################################################
+    # spin up dockers if not started yet
+    # - docker-compose up -d db django behave ws selenoid novnc
     # save all environment variables that start with COMETA_ to secret_variables.py
     # these variables come from CI/CD variables from gitlab and variables set inside
     # gitlab-ci yaml files.
-    - env | grep "^COMETA_" | sed -E "s/([A-Z_])=(.*)/\1='\2'/" > secret_variables.py
+    - env | grep "^COMETA_" | sed -E "s/([A-Z_])=(.*)/\1='\2'/" > backend/secret_variables.py
     # Build Selenoid browsers
     - echo -e "\e[0Ksection_start:`date +%s`:deploy_selenoid\r\e[0KRunning selenoid deployment script"
     - bash backend/selenoid/deploy_selenoid.sh
@@ -68,52 +88,50 @@ stages:
     - echo -e "\e[0Ksection_start:`date +%s`:parse_browsers\r\e[0KParsing selenoid browsers to update database"
     - docker exec cometa_django bash -c "curl http://localhost:8000/parseBrowsers/ --silent --retry 5 --retry-delay 5"
     - echo -e "\e[0Ksection_end:`date +%s`:parse_browsers\r\e[0K"
-    # Reload project files in gunicorn
-    - echo -e "\e[0Ksection_start:`date +%s`:reloading_gunicorn\r\e[0KPreparing to reload project in gunicorn"
-    - echo -e "Now reloading gunicorn for django"
-    - docker exec cometa_django fuser -k -HUP 8000/tcp
-    - echo -e "Now reloading gunicorn for behave"
-    - docker exec cometa_behave fuser -k -HUP 8001/tcp
-    - echo -e "\e[0Ksection_end:`date +%s`:reloading_gunicorn\r\e[0K"
 
     #########################################################
     # FRONTEND
     #########################################################
-    # Replace secrets in openidc
-    - sed -i 's/@@COMETA_CRYPTO_PASSPHRASE@@/'$COMETA_CRYPTO_PASSPHRASE'/g' front/apache-conf/openidc.conf_${DOCKER_OPENIDC_CONFIG_EXT}
-    # Replace google client information
-    - sed -i 's/@@COMETA_GOOGLE_CLIENTID@@/'$COMETA_GOOGLE_CLIENTID'/g' front/apache-conf/metadata/accounts.google.com.client
-    - sed -i 's/@@COMETA_GOOGLE_SECRETKEY@@/'$COMETA_GOOGLE_SECRETKEY'/g' front/apache-conf/metadata/accounts.google.com.client
-    # Replace Stripe client keys
-    - sed -i 's/@@COMETA_STRIPE_PUBLIC_LIVE_KEY@@/'$COMETA_STRIPE_PUBLIC_LIVE_KEY/'g' front/src/app/deploy-tokens.ts
-    - sed -i 's/@@COMETA_STRIPE_PUBLIC_TEST_KEY@@/'$COMETA_STRIPE_PUBLIC_TEST_KEY'/g' front/src/app/deploy-tokens.ts
-    # update docker-compose file depending on server variables
-    - sed -i 's/<outside_port>/'$DOCKER_HTTP_PORT'/g;s/<server>/'$DOCKER_OPENIDC_CONFIG_EXT'/g' docker-compose.yml
-    # Replace co.meta favicon depending on the branch
-    - for FILE in ${COMETA_REPLACE_FAVICON_IN}; do sed -i 's/@@BRANCH@@/'$CI_COMMIT_BRANCH'/g' $FILE; done
     # start container or if already started then compile else start from zero
     - echo "Checking if cometa_front is already running else start it..."
     - echo "If there are changes to docker-compose file it will recreate the container..."
-    - docker-compose up -d apache 2>&1 | grep "up-to-date" && docker exec cometa_front bash -c "cd /code/; ./start.sh $script_options" || ( timeout 360 docker-compose logs -f apache || true )
+    - docker-compose up -d apache 2>&1 | grep -E "up-to-date|cometa_front.*?Running" && docker exec cometa_front bash -c "cd /code/; ./start.sh $script_options" || ( timeout 360 docker-compose logs -f --tail=0 apache || true )
+
+    #########################################################
+    # RELOAD GUNICORNS
+    #########################################################
+    # Reload project files in gunicorn
+    - echo -e "\e[0Ksection_start:`date +%s`:reloading_gunicorn\r\e[0KPreparing to reload project in gunicorn"
+    - echo -e "Now reloading gunicorn for django"
+    - docker exec cometa_django fuser -k -HUP 8000/tcp
+    - echo -e "Now reloading gunicorn for behave"
+    - docker exec cometa_behave fuser -k -HUP 8001/tcp
+    - docker exec cometa_behave bash -c "supervisorctl restart django-rq:"
+    - echo -e "\e[0Ksection_end:`date +%s`:reloading_gunicorn\r\e[0K"
 
 # create a deploy job for amvara2 for branch stage
 deploy:amvara2.stage:
   <<: *deployment_script
   extends: .amvara2.stage
 
 # create a deploy job for amvara3 for branch prod
-# deploy:amvara3.prod:
-#   <<: *deployment_script
-#   extends: .amvara3.prod
+deploy:amvara3.prod:
+  <<: *deployment_script
+  extends: .amvara3.prod
 
 # create a deploy job for client01 for branch prod
-# deploy:client01.int:
-#   <<: *deployment_script
-#   extends: .client01.int
+deploy:client01.int:
+  <<: *deployment_script
+  extends: .client01.int
 
 # create a deploy job for client01 for branch prod
-# deploy:client01.prod:
+deploy:client01.prod:
+  <<: *deployment_script
+  extends: .client01.prod
+
+# create a deploy job for client02 for branch prod
+# deploy:client02.prod:
 #   <<: *deployment_script
-#   extends: .client01.prod
+#   extends: .client02.prod
 
 # DONE

README.md

@@ -4,12 +4,10 @@
 <p  align="center">
 <h1  align="center">Cometa</h1>
 <p  align="center">
-COMETA is a 100% open source software suite for visual and functional regression testing, to help QA Managers, DevOps and Business Owners get rid of repeating manual tests. <a  href="https://cometa.rocks/"><strong>Learn more</strong></a>
+Open source is the future. Co.Meta is an advanced & evolving meta-test product that has been made with ❤️ for DevOps and QA Engineers. <a  href="https://cometa.rocks/"><strong>Learn more</strong></a>
 <br>
 <br>
-<a  href="https://t.me/joinchat/bFquCBGPBCAwYWZk">Telegram</a>
-·
-<a href="https://matrix.to/#/!LqMzSzyPqrUFRqNiCt:matrix.org?via=matrix.org">Matrix</a>
+<a  href="https://cometa.rocks/support/">Support</a>
 
 </p>
 
@@ -24,11 +22,7 @@ COMETA is a 100% open source software suite for visual and functional regression
 - [Django](https://www.djangoproject.com/)
 - [Behave](https://behave.readthedocs.io/en/stable/)
 - [Selenoid](https://aerokube.com/selenoid/)
-
-## Requirements
-
-- Docker
-- Docker Compose
+- [PostgreSQL](https://www.postgresql.org/)
 
 ## Getting started
 
@@ -38,8 +32,9 @@ Here is what you need to be able to run Cometa.
 
 - Docker
 - Docker Compose
+- Use Linux as operating system. Have a look at pre-build [Virtual Boxes](https://osboxes.org/)
 
-In any case that you are stuck for more than 5 minutes - please us know. And please give us the oportunity to help you. We want to learn how you are using cometa and what problems you encounter. Contact us via Telegram or WhatsApp as seen on https://cometa.rocks/ .
+In any case that you are stuck for more than 5 minutes - please us know. And please give us the oportunity to help you. We want to learn how you are using cometa and what problems you encounter. <a href="https://cometa.rocks/support/">Contact us</a>. We are happy to help.
 
 #### Manual
 
@@ -49,67 +44,109 @@ In any case that you are stuck for more than 5 minutes - please us know. And ple
    ```
 
 2. Setup at least 1 authentication provider:
+
 	To setup Google:
 	* Go to [Google Developer Console](https://console.cloud.google.com/)
 	* Create an OAuth application
 	* Add your domain to the allowed hosts
 	* Retrieve the `client_id` and `secret_id` and paste them in `./front/apache-conf/metadata/accounts.google.com.client`
 
+	* Set `redirect_uri` to `https://<domain>/callback`
+
+	To setup Gitlab:
+	* Goto [git.amvara.de](https://git.amvara.de/)
+	* Create a new account
+	* Settings > Application > Add new application
+	* Add your domain to the allowed hosts
+	* Retrieve the `client_id` and `secret_id` and paste them in `./front/apache-conf/metadata/accounts/git.amvara.de.client`
+
+	* Set `redirect_uri` to `https://<domain>/callback`
+
+	In both cases, the default URL when installing on you Desktop or Laptop, is `localhost`.
+
+
+Very nice: Instead of following the manual setup instructions below, you may execute `./cometa.sh` to bring up a localhost version on your machine.
+
+
+3. Create a crontab file for scheduling your automated tests
+
+   ```sh
+   mkdir -p backend/behave/schedules && touch backend/behave/schedules/crontab
+   ```
+
+4. Get all Docker containers up:
+
+	In `docker-compose.yml` change <server> to 'local' and <server-outside-port> to '80' or according to your needs.
+
+	* Change the `<outside_port>` port to `80` or any other port you'd like. Keep in mind that this port should match with what is configured inside the `openidc.conf`
+	* Change the `<server>` to `local` or your custom configuration file in `front/apache-conf/openidc.conf_<custom_name>`
+
+
 
-3. Get all Docker containers up:
 	```sh
-	docker-compose up -d
+	docker-compose up -d && docker-compose logs -f --tail=10
 	```
 
 	Cometa starts on port 443. If that port is used on your machine, change it `docker-compose.yml` to e.g. "8443:443"
 	Cometa also starts on port 80. If that is not available you could change that to 8081 ind `docker-compose.yml`
 
-	In case you want to view some logs `docker-compose logs -f --tail=10`
+	View some logs `docker-compose logs -f --tail=10` of the installation process, to get a understanding, when cometa is ready.
 
-4. Load required database objects
-	```bash
-	docker exec -it cometa_django bash
-	python manage.py loaddata defaults/*.json
+	Give cometa some minutes to install python, setup django, download npm and docker files, compile the front end.
+	Depending on your computer this can take a couple of minutes.
+
+	You want a development server?
+
+	```sh
+	docker-compose -f docker-compose-dev.yml up -d
 	```
 
-5. (optional) Create superuser for the Backend Admin:
+5. **(Optional)** Create superuser for the Backend Admin
 
-Default superuser is created on runtime as `admin:admin`
+	Default superuser is created on runtime as `admin:admin`.
 
-	```bash
+	```
+	bash
 	docker exec -it cometa_django bash
 	root@cometa_django:/opt/code# python manage.py createsuperuser
-	```
+	``` 
 
-6. Run the selenoid setup
+6. **(Optional)** Install latest browser versions
 
-`./selenoid/deploy_selenoid.sh`.
+	`./backend/selenoid/deploy_selenoid.sh -n 3`.
 
-This will configure and pull the Docker images for Selenoid.
+	This will configure and pull the three newest Docker images with virtual browsers for Selenoid.
 
-Selenoid image are the browser that you will be able use and select in cometa. 
+	Selenoid image are the browser that you will be able use and select in cometa. 
 
-Of course there are options to include browserstack, headspin or sourcelabs browsers. But that is a bit something you would not want to configure on your first setup.
+	Of course there are options to include browserstack, headspin or sourcelabs browsers. But that is a bit you would not want to configure on your first setup.
 
-This step will take some time as all the default browser images are being pulled.
+	This step will take some time as all the default browser images are being pulled.
+
+	Once cometa is up and running, you can parse the new browser images avaible into Cometa by calling `https://localhost/backend/parseBrowsers/`
 
 7. See cometa rocks in your browser
 
-Test server access `curl -k  https://<yourdomain>:<specified port - default 443>/`
+	Test server access `curl -k  https://<yourdomain>:<specified port - default 443>/`
+
+	Example `curl -k  https://localhost:443/`
+
+	You should see something like this:
+	<p>The document has moved <i>here</i>.</p>
 
-Example `curl -k  https://localhost:8443/`
+8. Import the over 70 pre-defined actions
 
-You should see something like this:
-<p>The document has moved <i>here</i>.</p>
+	On first start you have to manually parse the actions.py file. This enables cometa to use any steps defined in the UI. The user can then choose from the steps in the UI.
+	`https://localhost/backend/parseActions/` ... as a result cometa will show all actions that have been parsed and are now available for selection in cometa-front.
 
 
 8. Run your first test
 
-Click on the "+" on the very top. Select Department, Environment and Feature Name
+	Click on the "+" on the very top. Select Department, Environment and Feature Name
 
-And import this JSON to search for "cometa Rocks" on google
+	And import this JSON to search for "cometa Rocks" on google
 
-```[{"enabled":true,"screenshot":true,"step_keyword":"Given","compare":false,"step_content":"Goto URL \"https://www.google.de/\"","step_type":"normal","continue_on_failure":false,"timeout":60},{"enabled":true,"screenshot":false,"step_keyword":"Given","compare":false,"step_content":"Maximize the browser","step_type":"normal","continue_on_failure":false,"timeout":60},{"enabled":true,"screenshot":true,"step_keyword":"Given","compare":false,"step_content":"wait until I can see \"google\" on page","step_type":"normal","continue_on_failure":false,"timeout":60},{"enabled":true,"screenshot":true,"step_keyword":"Given","compare":false,"step_content":"I move mouse to \"//button[2]\" and click","step_type":"normal","continue_on_failure":true,"timeout":5},{"enabled":true,"screenshot":true,"step_keyword":"Given","compare":false,"step_content":"I move mouse to \"//input\" and click","step_type":"normal","continue_on_failure":false,"timeout":60},{"enabled":true,"screenshot":true,"step_keyword":"Given","compare":false,"step_content":"Send keys \"cometa rocks\"","step_type":"normal","continue_on_failure":false,"timeout":60},{"enabled":true,"screenshot":true,"step_keyword":"Given","compare":false,"step_content":"Press Enter","step_type":"normal","continue_on_failure":false,"timeout":60},{"enabled":true,"screenshot":true,"step_keyword":"Given","compare":false,"step_content":"wait until I can see \"cometa rocks\" on page","step_type":"normal","continue_on_failure":false,"timeout":60},{"enabled":true,"screenshot":true,"step_keyword":"Given","compare":true,"step_content":"I sleep \"1\" seconds","step_type":"normal","continue_on_failure":false,"timeout":60}]```
+	```[{"enabled":true,"screenshot":true,"step_keyword":"Given","compare":false,"step_content":"Goto URL \"https://www.google.de/\"","step_type":"normal","continue_on_failure":false,"timeout":60},{"enabled":true,"screenshot":false,"step_keyword":"Given","compare":false,"step_content":"Maximize the browser","step_type":"normal","continue_on_failure":false,"timeout":60},{"enabled":true,"screenshot":true,"step_keyword":"Given","compare":false,"step_content":"wait until I can see \"google\" on page","step_type":"normal","continue_on_failure":false,"timeout":60},{"enabled":true,"screenshot":true,"step_keyword":"Given","compare":false,"step_content":"I move mouse to \"//button[2]\" and click","step_type":"normal","continue_on_failure":true,"timeout":5},{"enabled":true,"screenshot":true,"step_keyword":"Given","compare":false,"step_content":"I move mouse to \"//input\" and click","step_type":"normal","continue_on_failure":false,"timeout":60},{"enabled":true,"screenshot":true,"step_keyword":"Given","compare":false,"step_content":"Send keys \"cometa rocks\"","step_type":"normal","continue_on_failure":false,"timeout":60},{"enabled":true,"screenshot":true,"step_keyword":"Given","compare":false,"step_content":"Press Enter","step_type":"normal","continue_on_failure":false,"timeout":60},{"enabled":true,"screenshot":true,"step_keyword":"Given","compare":false,"step_content":"wait until I can see \"cometa rocks\" on page","step_type":"normal","continue_on_failure":false,"timeout":60},{"enabled":true,"screenshot":true,"step_keyword":"Given","compare":true,"step_content":"I sleep \"1\" seconds","step_type":"normal","continue_on_failure":false,"timeout":60}]```
 
 
 #### Notes
@@ -155,10 +192,10 @@ That's all, easy peasy.
 
 ## License
 
-Copyright 2021 COMETA ROCKS S.L.
+Copyright 2022 COMETA ROCKS S.L.
 
 Portions of this software are licensed as follows:
 
-* All content that resides under "ee/" directory of this repository (Enterprise Edition) is licensed under the license defined in "ee/LICENSE".
+* All content that resides under "ee/" directory of this repository (Enterprise Edition) is licensed under the license defined in "ee/LICENSE". (Work in progress)
 * All third party components incorporated into the cometa.rocks Software are licensed under the original license provided by the owner of the applicable component.
 * Content outside of the above mentioned directories or restrictions above is available under the "AGPLv3" license as defined in `LICENSE` file.

backend/.gitignore

@@ -19,12 +19,13 @@ backups/
 migrations/
 downloads/
 videos/
-schedules/
+pdf/
 
 crontab
 
 # department folders
 department_data/
+uploads/
 
 # logs
 logs/
@@ -35,4 +36,7 @@ output.txt
 accounts.google.com.client
 .vars.sh
 secret_variables.py
-.initiated
+.initiated
+
+# browsers.json from selenoid
+browsers.json