diff --git a/.dockerignore b/.dockerignore
index bd234b75..10525905 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,25 +1,31 @@
-results
-scr
-old*
-keys
-logs
-data
-backup*
-test_run
-htmlcov
-docs
-bulma
-scratch
+!bin/*.tar.bz2
*.override.yml
-*__pycache__*
-*.swp
+*.pem
*.swo
-frontend/tests/keys
-frontend/tests/scr
-frontend/tests/results
+*.swp
+*__pycache__*
.git
+backup*
bin/*
-!bin/*.tar.bz2
-gcloud.sh
+bulma
+data
+docs
frontend/cloud_config.py
-*.pem
+frontend/tests/keys
+frontend/tests/results
+frontend/tests/scr
+gcloud.sh
+htmlcov
+keys
+logs
+npm_modules/node_modules
+npm_packages/css
+npm_packages/css-dist
+npm_packages/js-dist
+npm_packages/node_modules
+old*
+results
+scr
+scratch
+test_run
+firebase
diff --git a/.gitignore b/.gitignore
index 2b411bdf..0a7acc78 100755
--- a/.gitignore
+++ b/.gitignore
@@ -1,27 +1,34 @@
+!.github
+!bin/*.tar.bz2
+*.override.yml
+*.pem
*.pyc
-*__pycache__
+*.sqlite3
*.swp
-calcus/*settings*
+*__pycache__
*keys/
-*scr/
*results/
-static/CACHE/
-frontend/migrations/
-*.sqlite3
+*scr/
+scratch/
.*
-!.github
-*.override.yml
-htmlcov/
-old/
-data/
-logs/
backup*
-test_run
-docs/build/
-celerybeat-schedule
bin/*
-!bin/*.tar.bz2
-gcloud.sh
-frontend/cloud_config.py
+calcus/*settings*
+celerybeat-schedule
+data/
+docs/build/
frontend/cloud_config.json
-*.pem
+frontend/cloud_config.py
+frontend/migrations/
+gcloud.sh
+htmlcov/
+logs/
+npm_packages/css
+npm_packages/css-dist
+npm_packages/js-dist
+npm_packages/node_modules
+old/
+static/CACHE/
+static/testimonials
+test_run
+firebase
diff --git a/Cloud_dockerfile b/Cloud_dockerfile
index 6171ad0f..ec0853e6 100644
--- a/Cloud_dockerfile
+++ b/Cloud_dockerfile
@@ -8,7 +8,7 @@ ENV CALCUS_CLOUD True
ADD ./cloud_requirements.txt /calcus/cloud_requirements.txt
RUN pip install -r /calcus/cloud_requirements.txt
-RUN apt update && apt install openbabel postgresql-client dos2unix -y
+RUN apt update && apt install openbabel postgresql-client dos2unix nwchem -y
COPY calcus /calcus/calcus
COPY frontend /calcus/frontend
diff --git a/Dockerfile b/Dockerfile
index 531c035a..53c4308b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.9 AS calcus_user
+FROM python:3.9-bullseye AS calcus_user
ARG CALCUS_VERSION_HASH
ENV CALCUS_VERSION_HASH=${CALCUS_VERSION_HASH}
@@ -20,7 +20,10 @@ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:"/binaries/orca"
ENV PATH=$PATH:$XTB4STDAHOME/xtb/bin:$XTB4STDAHOME:$EBROOTORCA:$GAUSS_EXEDIR
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/binaries/orca:/usr/lib/openmpi/
-RUN apt update && apt install openbabel sshpass postgresql-client dos2unix openmpi-bin -y
+RUN apt update && apt install openbabel sshpass postgresql-client dos2unix python3-dev gfortran mpi-default-bin mpi-default-dev -y
+RUN curl -LJO https://github.com/nwchemgit/nwchem/releases/download/v7.2.0-release/nwchem-data_7.2.0-2_all.debian_bullseye.deb
+RUN curl -LJO https://github.com/nwchemgit/nwchem/releases/download/v7.2.0-release/nwchem_7.2.0-2_amd64.debian_bullseye.deb
+RUN dpkg -i nwchem*7.2.0*.deb
ADD ./requirements.txt /calcus/requirements.txt
RUN pip install -r /calcus/requirements.txt
diff --git a/README.md b/README.md
index 5f0b52ea..0173a042 100644
--- a/README.md
+++ b/README.md
@@ -72,12 +72,13 @@ If you use CalcUS in your research, please cite it:
**Contributor of significant features**:
-+ Divyam Agarwal (automated multi-step calculations)
++ Divyam Agarwal (@div-yam): automated multi-step calculations (partially completed)
**Contributors to the web frontend**:
-+ Glen Forcha
-+ Dhairya Khanna
++ Glen Forcha (@ForchaGlen)
++ Dhairya Khanna (@Dhairya3124)
++ Omkar (@Omkar0803)
**Beta-testers**:
diff --git a/bin/xtb_suite.tar.bz2 b/bin/xtb_suite.tar.bz2
index 848effbd..126c4681 100644
Binary files a/bin/xtb_suite.tar.bz2 and b/bin/xtb_suite.tar.bz2 differ
diff --git a/calcus/settings.py b/calcus/settings.py
index 732407c2..79f135ae 100755
--- a/calcus/settings.py
+++ b/calcus/settings.py
@@ -67,8 +67,6 @@
stripe.api_key = STRIPE_SECRET_KEY
-SUBSCRIPTION_ACADEMIC_MONTHLY = os.getenv("SUBSCRIPTION_ACADEMIC_MONTHLY", "")
-
# On Google Cloud, the URI of the secret containing the PostgreSQL password (for compute instances)
POSTGRES_SECRET_URI = os.getenv("POSTGRES_SECRET_URI", "")
@@ -76,21 +74,24 @@
POSTGRES_HOST = os.environ.get("POSTGRES_HOST", "postgres")
POSTGRES_USER = os.environ.get("POSTGRES_USER", "calcus")
+"""
GMAIL_API_CLIENT_ID = os.getenv("CALCUS_EMAIL_ID", "")
GMAIL_API_CLIENT_SECRET = os.getenv("CALCUS_EMAIL_SECRET", "")
GMAIL_API_REFRESH_TOKEN = os.getenv("CALCUS_EMAIL_TOKEN", "")
+"""
ALLOWED_HOSTS = [
"0.0.0.0",
"0.0.0.0:*",
"https://calcus.cloud",
+ "https://static.calcus.cloud",
"calcus.cloud",
- "localhost",
+ "static.calcus.cloud" "localhost",
"cloud-compute",
"cloud-compute:*",
]
-CSRF_TRUSTED_ORIGINS = ["calcus.cloud"]
+CSRF_TRUSTED_ORIGINS = ["calcus.cloud", "static.calcus.cloud"]
if "CALCUS_CLOUD_INTERNAL" in os.environ:
ALLOWED_HOSTS.append("*")
@@ -116,14 +117,15 @@
"django.contrib.sitemaps",
"axes",
"bulma",
- "gmailapi_backend",
+ # "gmailapi_backend",
"corsheaders",
#'debug_toolbar',
]
-if IS_CLOUD:
+if IS_CLOUD or IS_TEST:
INSTALLED_APPS.append("captcha")
+if IS_CLOUD:
if not DEBUG:
SECURE_SSL_REDIRECT = True
SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
@@ -164,6 +166,8 @@
"https://optimize.google.com",
"https://fonts.googleapis.com",
"https://storage.googleapis.com",
+ "https://static.calcus.cloud",
+ "https://www.static.calcus.cloud",
]
HASHID_FIELD_SALT = os.getenv("CALCUS_HASHID_SALT", "test_salt")
@@ -259,7 +263,15 @@
EMAIL_BACKEND = "django.core.mail.backends.filebased.EmailBackend"
EMAIL_FILE_PATH = os.path.join(BASE_DIR, "scratch", "sent_emails")
else:
- EMAIL_BACKEND = "gmailapi_backend.mail.GmailBackend"
+ # EMAIL_BACKEND = "gmailapi_backend.mail.GmailBackend"
+ EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend"
+ EMAIL_HOST = "smtp.gmail.com"
+ EMAIL_PORT = 587
+ EMAIL_USE_TLS = True
+ EMAIL_USE_SSL = False
+ EMAIL_HOST_USER = os.getenv("CALCUS_EMAIL_USER")
+ EMAIL_HOST_PASSWORD = os.getenv("CALCUS_EMAIL_PASSWORD")
+
MEDIA_URL = "/media/"
@@ -300,7 +312,7 @@
},
"subscriber": {
"nproc": 8,
- "time": 120,
+ "time": 6 * 60,
},
}
@@ -308,6 +320,11 @@
TRIAL_DEFAULT_COMP_SECONDS = 600
FREE_DEFAULT_COMP_SECONDS = 3600
+# These allocation amounts shouldn't be used as official reference anymore.
+# The real amounts are parsed from Stripe. These are for testing
+SUBSCRIBER_COMP_SECONDS = 3600 * 60
+SUBSCRIBER_TEAM_COMP_SECONDS = 3600 * 500
+
if IS_CLOUD:
PING_SATELLITE = False
@@ -325,14 +342,14 @@
ALLOW_TRIAL = True
- LOCAL_MAX_ATOMS = 200
+ LOCAL_MAX_ATOMS = 50000
LOCAL_ALLOWED_THEORY_LEVELS = [
"xtb",
# "semiempirical",
- # "hf",
+ "hf",
# "special", # hf3c, pbeh3c, r2scan3c, b973c
- # "dft",
+ "dft",
# "mp2",
# "cc",
]
@@ -340,15 +357,17 @@
LOCAL_ALLOWED_STEPS = [
"Geometrical Optimisation",
"Conformational Search",
+ "Fast Conformational Search",
"Constrained Optimisation",
"Frequency Calculation",
"TS Optimisation",
"UV-Vis Calculation",
"Single-Point Energy",
- # "Minimum Energy Path",
+ "Minimum Energy Path",
"Constrained Conformational Search",
# "NMR Prediction",
- # "MO Calculation",
+ "MO Calculation",
+ "ESP Calculation",
]
else:
diff --git a/calcus/sitemaps.py b/calcus/sitemaps.py
index f7a3439a..37e7bba3 100644
--- a/calcus/sitemaps.py
+++ b/calcus/sitemaps.py
@@ -7,7 +7,16 @@ class CalcUSSitemap(sitemaps.Sitemap):
changefreq = "monthly"
def items(self):
- return ["frontend:home", "frontend:pricing"]
+ return [
+ ("frontend:home", {}),
+ ("frontend:pricing", {}),
+ ("frontend:register", {}),
+ ("frontend:login", {}),
+ ("frontend:start_trial", {}),
+ ("frontend:learn", {}),
+ ("frontend:learn_keyword", {"keyword": "conformational_search"}),
+ ]
- def location(self, item):
- return reverse(item)
+ def location(self, item_data):
+ item, kwargs = item_data
+ return reverse(item, kwargs=kwargs)
diff --git a/calcus/urls.py b/calcus/urls.py
index 757fd1ac..9abdd123 100755
--- a/calcus/urls.py
+++ b/calcus/urls.py
@@ -27,6 +27,7 @@
from django.contrib.sitemaps.views import sitemap
from .sitemaps import CalcUSSitemap
+from .views import privacy
sitemaps = {
"static": CalcUSSitemap,
@@ -34,6 +35,7 @@
urlpatterns = [
path("", include("frontend.urls", namespace="frontend")),
+ path("privacy/", privacy),
path("accounts/", include("django.contrib.auth.urls")),
path(
"sitemap.xml",
diff --git a/calcus/views.py b/calcus/views.py
index b727e50b..b09440c4 100755
--- a/calcus/views.py
+++ b/calcus/views.py
@@ -45,3 +45,7 @@ class LogoutView(TemplateView):
def get(self, request, **kwargs):
logout(request)
return render(request, self.template_name)
+
+
+def privacy(request):
+ return render(request, "registration/privacy.html")
diff --git a/cloud_requirements.txt b/cloud_requirements.txt
index 04e0eedf..ac4a6089 100644
--- a/cloud_requirements.txt
+++ b/cloud_requirements.txt
@@ -1,6 +1,6 @@
basis-set-exchange
bleach
-ccinput>=1.7.1
+ccinput>=1.9.1
cryptography
Django>=3.2.18,<4
django-appconf==1.0.4
@@ -23,8 +23,11 @@ periodictable
psutil==5.7.2
psycopg2
pysisyphus
+rdkit
requests
scipy
+scikit-learn
+spyrmsd
stripe
wheel
xkcdpass
diff --git a/dev-compose.yml b/dev-compose.yml
index 2c0054f3..cb376063 100644
--- a/dev-compose.yml
+++ b/dev-compose.yml
@@ -45,7 +45,7 @@ services:
context: .
dockerfile: Dockerfile
target: calcus_user
- user: calcus
+ user: "${UID}:${GID}"
command: ./scripts/run_celery_main.sh
volumes:
- ./scr:/calcus/scr
@@ -61,7 +61,7 @@ services:
context: .
dockerfile: Dockerfile
target: calcus_user
- user: calcus
+ user: "${UID}:${GID}"
shm_size: 1gb
command: ./scripts/run_celery_comp.sh
volumes:
@@ -78,7 +78,7 @@ services:
context: .
dockerfile: Dockerfile
target: calcus_user
- user: calcus
+ user: "${UID}:${GID}"
command: ./scripts/run_cluster_daemon.sh
env_file:
- ./.env
diff --git a/docker-compose.yml b/docker-compose.yml
index 71177eb2..49a417e6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -40,7 +40,7 @@ services:
- web
celery_main:
image: raphaelrobidas/calcus2:latest
- user: calcus
+ user: "${UID}:${GID}"
command: ./scripts/run_celery_main.sh
volumes:
- ./scr:/calcus/scr
@@ -54,7 +54,7 @@ services:
- redis
celery_comp:
image: raphaelrobidas/calcus2:latest
- user: calcus
+ user: "${UID}:${GID}"
command: ./scripts/run_celery_comp.sh
volumes:
- ./scr:/calcus/scr
diff --git a/docs/source/software-packages.rst b/docs/source/software-packages.rst
index 50e7ef35..218b6849 100644
--- a/docs/source/software-packages.rst
+++ b/docs/source/software-packages.rst
@@ -70,7 +70,7 @@ The Hamiltonian can be chosen with ``--gfn 2`` (default), ``--gfn 1``, ``--gfn 0
The convergence criteria of geometrical optimisations can be chosen with ``--opt level``, where level is ``crude``, ``sloppy``, ``loose``, ``lax``, ``normal``, ``tight`` (default), ``vtight`` or ``extreme``.
-Conformational searches (constrained or not) have several particular options. Faster/cruder sampling procedures can be requested with ``--quick``, ``--squick`` and ``--mquick``. The RMSD threshold for considering conformers as different can be set with ``--rthr X``, where ``X`` is the threshold in Ångström (default of 0.6 in CalcUS). Furthermore, the energy window to consider can be set with ``--ewin X``, where ``X`` is the treshold in kcal/mol (default of 6 in CalcUS).
+Conformational searches (constrained or not) have several particular options. Faster/cruder sampling procedures can be requested with ``--quick``, ``--squick`` and ``--mquick``. The RMSD threshold for considering conformers as different can be set with ``--rthr X``, where ``X`` is the threshold in Ångström (default of 2.0 in CalcUS). Furthermore, the energy window to consider can be set with ``--ewin X``, where ``X`` is the treshold in kcal/mol (default of 6 in CalcUS).
Constrained geometrical optimisations and constrained conformational searches employ a harmonic potential to constrain coordinates (distances, angles, dihedral angles). The force constant of this potential can be chosen with ``--forceconstant X``, where ``X`` is the force constant in Hartree/Bohr². By default, CalcUS uses a force constant of 1.0, which corresponds to a very stiff potential well. In most cases, this high value forces the use of small timesteps in meta-dynamic and molecular dynamic simulations (for constrained conformational searches). If the constrained coordinates do not need to remain exactly constant, the simulations can be accelerated by a smaller force constant.
diff --git a/frontend/calcusliveserver.py b/frontend/calcusliveserver.py
index 9b0a1e13..f8ecab00 100644
--- a/frontend/calcusliveserver.py
+++ b/frontend/calcusliveserver.py
@@ -37,7 +37,6 @@
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import Select
from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.common.action_chains import ActionChains
from celery.contrib.testing.worker import start_worker
@@ -78,18 +77,19 @@ def setUpClass(cls):
cls.host = socket.gethostbyname(socket.gethostname())
+ chrome_options = Options()
if "CI" in os.environ: # Github Actions
- chrome_options = Options()
- from pyvirtualdisplay import Display
+ chrome_options.add_argument("--headless")
+ # from pyvirtualdisplay import Display
- cls.display = Display(visible=0, size=(ZOOM * 1920, ZOOM * 1080))
- cls.display.start()
+ # cls.display = Display(visible=0, size=(ZOOM * 1920, ZOOM * 1080))
+ # cls.display.start()
- cls.driver = webdriver.Chrome(chrome_options=chrome_options)
+ cls.driver = webdriver.Chrome(options=chrome_options)
else:
cls.driver = webdriver.Remote(
command_executor="http://selenium:4444/wd/hub",
- desired_capabilities=DesiredCapabilities.CHROME,
+ options=chrome_options,
)
cls.driver.set_window_size(ZOOM * 1920, ZOOM * 1080)
@@ -151,7 +151,7 @@ def _test_wrapper(self):
num += 1
print(f"Test failed, trying again (attempt {num}/{MAX_ATTEMPTS})")
self.cleanupCalculations()
- self.lget("/home/")
+ self.lget("/")
time.sleep(3)
else:
break
@@ -163,7 +163,14 @@ def setUp(self):
self.email = "Selenium@test.com"
self.password = "test1234"
- self.user = User.objects.create_user(email=self.email, password=self.password)
+ self.user = User.objects.create_user(
+ email=self.email,
+ password=self.password,
+ advanced_interface=True,
+ calc_type_property=False,
+ calc_method_suggestions=False,
+ )
+
self.login(self.email, self.password)
time.sleep(0.1) # Reduces glitches (I think?)
@@ -174,8 +181,9 @@ def setUp(self):
def cleanupCalculations(self):
for c in Calculation.objects.all():
- res = AbortableAsyncResult(c.task_id)
- res.abort()
+ if c.task_id != "":
+ res = AbortableAsyncResult(c.task_id)
+ res.abort()
def login(self, email, password):
self.lget("/accounts/login/")
@@ -234,6 +242,10 @@ def lget(self, url):
def calc_input_params(self, params):
self.wait_for_ajax()
+ simple_interface = False
+ if "interface" in params and params["interface"].lower() == "simple":
+ simple_interface = True
+
if "mol_name" in params.keys():
element = WebDriverWait(self.driver, 6).until(
EC.presence_of_element_located((By.NAME, "calc_mol_name"))
@@ -302,13 +314,14 @@ def calc_input_params(self, params):
else:
driver = "xtb"
- select = self.driver.find_element(By.NAME, "calc_driver")
- self.driver.execute_script(
- "showDropdown = function (element) {var event; event = document.createEvent('MouseEvents'); event.initMouseEvent('mousedown', true, true, window); element.dispatchEvent(event); }; showDropdown(arguments[0]);",
- select,
- )
- time.sleep(0.1)
- select.find_element(By.XPATH, f"option[text()='{driver}']").click()
+ if not simple_interface:
+ select = self.driver.find_element(By.NAME, "calc_driver")
+ self.driver.execute_script(
+ "showDropdown = function (element) {var event; event = document.createEvent('MouseEvents'); event.initMouseEvent('mousedown', true, true, window); element.dispatchEvent(event); }; showDropdown(arguments[0]);",
+ select,
+ )
+ time.sleep(0.1)
+ select.find_element(By.XPATH, f"option[text()='{driver}']").click()
self.wait_for_ajax()
@@ -962,11 +975,11 @@ def is_on_page_order_details(self):
def is_on_page_projects(self):
for i in range(3):
url = self.get_split_url()
- if (url[0] == "projects" or url[0] == "home") and (
- url[1] == "" or self.is_user(url[1])
+ if url[0] == "projects" and (
+ len(url) < 2 or self.is_user(url[1]) or url[1] == ""
):
return True
- time.sleep(1)
+ time.sleep(0.5)
return False
@@ -1519,6 +1532,10 @@ def launch_frame_next_step(self):
else:
raise Exception("Could not get the button to launch calculation from frame")
+ button = WebDriverWait(self.driver, 1).until(
+ EC.element_to_be_clickable((By.ID, "launch_from_frame"))
+ )
+
button.click() # Less flaky than "send_keys(Keys.RETURN)" in headless mode?
def accept_alert(self):
@@ -1679,7 +1696,12 @@ def setup_test_group(self):
g = ResearchGroup.objects.create(name="Test group", PI=self.user)
self.student = User.objects.create_user(
- email="Student@test.com", password=self.password, member_of=g
+ email="Student@test.com",
+ password=self.password,
+ member_of=g,
+ advanced_interface=True,
+ calc_type_property=False,
+ calc_method_suggestions=False,
)
def is_loaded_frequencies(self):
@@ -1932,11 +1954,22 @@ def clean_all_completed(self):
btn = self.driver.find_element(By.ID, "clean_all_completed_btn")
btn.click()
+ def load_mol_in_sketcher(self, mol):
+ # The mol text should not have any excess whitespace (e.g., due to indentation in the Python code)
+ self.driver.execute_script(
+ "sketcher.loadMolecule(ChemDoodle.readMOL(`" + mol + "`))"
+ )
+
+ def preview_molecule(self):
+ self.driver.find_element(By.ID, "gen_3D_btn").click()
+ self.wait_for_ajax()
+
class CalcusCloudLiveServer(CalcusLiveServer):
def setUp(self):
super().setUp()
self.user.allocated_seconds = 100
+ self.user.calc_type_property = False
self.user.save()
settings.IS_CLOUD = True
@@ -1966,6 +1999,45 @@ def tearDown(self):
settings.LOCAL_ALLOWED_STEPS = ["ALL"]
+ def register(self, acc_type, email, password, opt_in_emails):
+ self.lget("/register/")
+
+ # Researcher only for now
+ tab = WebDriverWait(self.driver, 2).until(
+ EC.presence_of_element_located((By.ID, "tab_" + acc_type))
+ )
+
+ tab.click()
+ if acc_type == "researcher":
+ self.driver.find_element(By.ID, "id_email").send_keys(email)
+ self.driver.find_element(By.ID, "id_password1").send_keys(password)
+ self.driver.find_element(By.ID, "id_password2").send_keys(password)
+ if opt_in_emails:
+ self.driver.find_element(By.ID, "id_opted_in_emails").click()
+
+ try:
+ WebDriverWait(self.driver, 3).until(
+ EC.frame_to_be_available_and_switch_to_it(
+ (By.XPATH, "//*[@title='reCAPTCHA']")
+ )
+ )
+ time.sleep(0.3)
+ WebDriverWait(self.driver, 1).until(
+ EC.element_to_be_clickable(
+ (By.XPATH, "//div[@class='recaptcha-checkbox-border']")
+ )
+ ).click()
+ self.wait_for_ajax()
+ self.driver.switch_to.default_content()
+ time.sleep(0.3)
+ except selenium.common.exceptions.NoSuchElementException:
+ pass
+
+ submit = self.driver.find_element(
+ By.CSS_SELECTOR, "#form_" + acc_type + " > button"
+ )
+ submit.click()
+
def redeem_code(self, code):
self.lget("/profile/")
@@ -2097,20 +2169,39 @@ def dissolve_class(self, name):
else:
raise Exception(f"Could not find a class named {name} to dissolve")
- def subscribe(self, email, card_number):
+ def subscribe(self, plan, email, card_number, length="month"):
+ """
+ Plan is either `researcher` or `team`
+ """
self.lget("/pricing/")
+ if length == "year":
+ switch = WebDriverWait(self.driver, 5).until(
+ EC.presence_of_element_located(
+ (By.CSS_SELECTOR, f"[for=sub_length_switch]")
+ )
+ )
+ switch.click()
+
btn = WebDriverWait(self.driver, 2).until(
- EC.element_to_be_clickable((By.ID, "subscribe_btn"))
+ EC.element_to_be_clickable((By.ID, f"subscribe_{plan}_btn_{length}"))
)
btn.click()
- email_inp = WebDriverWait(self.driver, 4).until(
- EC.presence_of_element_located((By.ID, "email"))
+ cardNumber = WebDriverWait(self.driver, 4).until(
+ EC.presence_of_element_located((By.ID, "cardNumber"))
)
- email_inp.send_keys(email)
+ cardNumber.send_keys(card_number)
+
+ # The email is autofilled if the user is logged in
+ try:
+ email_field = self.driver.find_element(By.ID, "email")
+ if not email_field.get_property("readonly"):
+ email_field.send_keys(email)
+ except selenium.common.exceptions.NoSuchElementException:
+ # It seems like the filled field does not have the same ID
+ pass
- self.driver.find_element(By.ID, "cardNumber").send_keys(card_number)
self.driver.find_element(By.ID, "cardExpiry").send_keys("0140")
self.driver.find_element(By.ID, "cardCvc").send_keys("123")
self.driver.find_element(By.ID, "billingName").send_keys("Selenium Robot")
diff --git a/frontend/cloud_job.py b/frontend/cloud_job.py
index 9967bc46..431e7d32 100644
--- a/frontend/cloud_job.py
+++ b/frontend/cloud_job.py
@@ -1,10 +1,12 @@
import os
import json
+import uuid
from django.conf import settings
from .models import Calculation
from .environment_variables import *
+from .helpers import job_triage
if settings.IS_CLOUD:
from google.cloud import batch_v1
@@ -31,18 +33,16 @@ def record_event_analytics(request, event_name, **extra_params):
):
return
- from google.cloud import tasks_v2
-
- client = tasks_v2.CloudTasksClient()
-
if "_ga" not in request.COOKIES:
logger.warning(
f"The Google Analytics cookie does not appear to be set for {request.user.id}"
)
return
+ client_id = ".".join(request.COOKIES["_ga"].split(".")[-2:])
+
payload = {
- "client_id": request.COOKIES["_ga"],
+ "client_id": client_id,
"events": [
{
"name": event_name,
@@ -54,10 +54,10 @@ def record_event_analytics(request, event_name, **extra_params):
],
}
- if "CALCUS_SESSION_COOKIE" in request.COOKIES:
- payload["events"][0]["params"]["session_id"] = request.COOKIES[
- "CALCUS_SESSION_COOKIE"
- ]
+ # Session ID
+ for cookie, val in request.COOKIES.items():
+ if "_ga_" in cookie and len(cookie) > 4:
+ payload["events"][0]["params"]["session_id"] = val.split(".")[2]
if not request.user.is_anonymous:
payload["user_id"] = str(request.user.id)
@@ -72,6 +72,13 @@ def record_event_analytics(request, event_name, **extra_params):
url = f"https://www.google-analytics.com/mp/collect?measurement_id={settings.ANALYTICS_MEASUREMENT_ID}&api_secret={settings.ANALYTICS_API_SECRET}"
+ if "gclid" in request.COOKIES:
+ url += f'&gclid={request.COOKIES["gclid"]}'
+
+ from google.cloud import tasks_v2
+
+ client = tasks_v2.CloudTasksClient()
+
parent = client.queue_path(
settings.GCP_PROJECT_ID, settings.GCP_LOCATION, "analytics"
)
@@ -113,8 +120,17 @@ def create_container_job(calc, nproc, timeout):
"CALCUS_CLOUD": "True",
"NUM_CPU": str(nproc),
"OMP_NUM_THREADS": f"{nproc},1",
- "OMP_STACKSIZE": "1536M", # for the c3 machines only
+ "OMP_STACKSIZE": "900M",
"CALCUS_TIMEOUT": str(timeout),
+ # Variables for rescheduling when a VM in preempted
+ "CALC_ID": str(calc.id),
+ "COMPUTE_IMAGE": settings.COMPUTE_IMAGE,
+ "COMPUTE_SERVICE_ACCOUNT": settings.COMPUTE_SERVICE_ACCOUNT,
+ "GCP_PROJECT_ID": settings.GCP_PROJECT_ID,
+ "GCP_LOCATION": settings.GCP_LOCATION,
+ # NWChem variables
+ "OMPI_ALLOW_RUN_AS_ROOT": "1",
+ "OMPI_ALLOW_RUN_AS_ROOT_CONFIRM": "1",
}
runnable.environment = env
@@ -124,7 +140,7 @@ def create_container_job(calc, nproc, timeout):
resources = batch_v1.ComputeResource()
resources.cpu_milli = nproc * 1000 # in milliseconds per cpu-second
- resources.memory_mib = nproc * 1024 * 2
+ resources.memory_mib = nproc * 1024
task.compute_resource = resources
task.max_retry_count = 1
@@ -138,10 +154,21 @@ def create_container_job(calc, nproc, timeout):
group.parallelism = 1
policy = batch_v1.AllocationPolicy.InstancePolicy()
- policy.machine_type = f"c3-highcpu-{nproc}"
+ policy.machine_type = f"n2d-highcpu-{nproc}"
+ # policy.boot_disk = ...
+ # https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#Disk
+
+ metadata = []
+
+ # Trying spot VMs
+ policy.provisioning_model = batch_v1.types.AllocationPolicy.ProvisioningModel.SPOT
+ metadata.append(
+ ("shutdown-script-url", "gs://calcus-cloud-config/shutdown_script.py")
+ )
instances = batch_v1.AllocationPolicy.InstancePolicyOrTemplate()
instances.policy = policy
+
acc = batch_v1.ServiceAccount()
acc.email = settings.COMPUTE_SERVICE_ACCOUNT
@@ -158,15 +185,16 @@ def create_container_job(calc, nproc, timeout):
job.logs_policy = batch_v1.LogsPolicy()
job.logs_policy.destination = batch_v1.LogsPolicy.Destination.CLOUD_LOGGING
+ rand = str(uuid.uuid4())[:4]
create_request = batch_v1.CreateJobRequest()
create_request.job = job
- create_request.job_id = f"j{str(calc.id).lower()}"
+ create_request.job_id = f"j{str(calc.id).lower()}{rand}"
create_request.parent = (
f"projects/{settings.GCP_PROJECT_ID}/locations/{settings.GCP_LOCATION}"
)
- return client.create_job(create_request)
+ return client.create_job(create_request, metadata=metadata)
def send_gcloud_task(url, payload, compute=True):
@@ -213,20 +241,6 @@ def send_gcloud_task(url, payload, compute=True):
client.create_task(parent=parent, task=task)
-def job_triage(calc):
- if calc.step.name in ["Conformational Search", "Constrained Conformational Search"]:
- nproc = 8
- else:
- nproc = 1
-
- user_type = calc.order.author.user_type
-
- return (
- min(settings.RESOURCE_LIMITS[user_type]["nproc"], nproc),
- settings.RESOURCE_LIMITS[user_type]["time"] * 60,
- )
-
-
def submit_cloud_job(calc):
nproc, timeout = job_triage(calc)
if nproc == 1:
diff --git a/frontend/constants.py b/frontend/constants.py
index 010ac157..a854f10e 100644
--- a/frontend/constants.py
+++ b/frontend/constants.py
@@ -47,6 +47,7 @@
R_CONSTANT = decimal.Decimal(8.31446261815324)
R_CONSTANT_HARTREE = decimal.Decimal(3.166811565240536e-06)
TEMP = decimal.Decimal(298)
+BOHR_VAL = 0.529177
MAX_SFTP_ATTEMPT_COUNT = 3
MAX_COMMAND_ATTEMPT_COUNT = 5
diff --git a/frontend/context.py b/frontend/context.py
index ec63a6f6..4b0b7005 100644
--- a/frontend/context.py
+++ b/frontend/context.py
@@ -1,18 +1,29 @@
from django.conf import settings
+from frontend import constants
def default(request):
- return {
- k: getattr(settings, k)
- for k in [
- "IS_TEST",
- "IS_CLOUD",
- "ALLOW_LOCAL_CALC",
- "ALLOW_REMOTE_CALC",
- "ALLOW_TRIAL",
- "LOCAL_MAX_ATOMS",
- "LOCAL_ALLOWED_THEORY_LEVELS",
- "LOCAL_ALLOWED_STEPS",
- "SUBSCRIPTION_ACADEMIC_MONTHLY",
- ]
- }
+ d = {}
+ for k in [
+ "IS_TEST",
+ "IS_CLOUD",
+ "ALLOW_LOCAL_CALC",
+ "ALLOW_REMOTE_CALC",
+ "ALLOW_TRIAL",
+ "LOCAL_MAX_ATOMS",
+ "LOCAL_ALLOWED_THEORY_LEVELS",
+ "LOCAL_ALLOWED_STEPS",
+ ]:
+ d[k] = getattr(settings, k)
+
+ if settings.IS_CLOUD:
+ if settings.IS_TEST:
+ version = "testing"
+ else:
+ version = "production"
+ try:
+ d["SUBSCRIPTION_DATA"] = constants.SUBSCRIPTION_DATA[version]
+ except (NameError, AttributeError):
+ logger.error(f"Could not get subscription data")
+
+ return d
diff --git a/frontend/decorators.py b/frontend/decorators.py
index 89cc7375..a0a84090 100644
--- a/frontend/decorators.py
+++ b/frontend/decorators.py
@@ -26,6 +26,6 @@ def wrapper(request, *args, **kwargs):
if request.user.is_superuser:
return func(request, *args, **kwargs)
else:
- return HttpResponseRedirect("/home/")
+ return HttpResponseRedirect("/")
return wrapper
diff --git a/frontend/forms.py b/frontend/forms.py
index d51b7755..84e5abc5 100644
--- a/frontend/forms.py
+++ b/frontend/forms.py
@@ -30,10 +30,10 @@
from django.contrib.auth.forms import SetPasswordForm
from django.utils import timezone
-if settings.IS_CLOUD:
+if settings.IS_CLOUD or settings.IS_TEST:
from captcha.fields import ReCaptchaField
-from frontend.models import User, ClassGroup, ResourceAllocation
+from frontend.models import User, ClassGroup, ResourceAllocation, Project
from frontend.helpers import get_random_string
logging.basicConfig(
@@ -49,7 +49,11 @@ class ResearcherCreateForm(UserCreationForm):
error_messages={"exists": "This email has already been used"},
)
- if settings.IS_CLOUD:
+ if settings.IS_CLOUD or settings.IS_TEST:
+ opted_in_emails = forms.BooleanField(
+ required=False,
+ label="Receive occasional emails about updates and promotions",
+ )
captcha = ReCaptchaField()
class Meta:
@@ -59,6 +63,10 @@ class Meta:
def save(self, commit=True):
user = super(ResearcherCreateForm, self).save(commit=False)
user.email = self.cleaned_data["email"]
+
+ if "opted_in_emails" in self.cleaned_data:
+ user.opted_in_emails = self.cleaned_data["opted_in_emails"]
+
user.allocated_seconds = settings.FREE_DEFAULT_COMP_SECONDS
user.last_free_refill = timezone.now()
@@ -70,6 +78,7 @@ def save(self, commit=True):
allocation_seconds=settings.FREE_DEFAULT_COMP_SECONDS,
note=ResourceAllocation.NEW_ACCOUNT,
)
+ proj = Project.objects.create(name="My Main Project", author=user)
return user
@@ -138,13 +147,16 @@ def save(self, commit=True):
if commit:
user.save()
+ proj = Project.objects.create(name="My Main Project", author=user)
return user
class TrialUserCreateForm(forms.ModelForm):
+ """
accepts_tos = forms.BooleanField(
required=True, label="I accept the Terms of Service"
)
+ """
if settings.IS_CLOUD:
captcha = ReCaptchaField()
@@ -153,12 +165,14 @@ class Meta:
model = User
fields = ()
+ """
def clean_accepts_tos(self):
accepts = self.cleaned_data["accepts_tos"]
if not accepts:
raise ValidationError(
f"You must accept the Terms of Service in order to use the platform"
)
+ """
def save(self, commit=True):
user = super().save(commit=False)
@@ -181,6 +195,7 @@ def save(self, commit=True):
allocation_seconds=settings.TRIAL_DEFAULT_COMP_SECONDS,
note=ResourceAllocation.TRIAL,
)
+ proj = Project.objects.create(name="My Main Project", author=user)
return user
diff --git a/frontend/helpers.py b/frontend/helpers.py
index e82756ca..d91a31ae 100644
--- a/frontend/helpers.py
+++ b/frontend/helpers.py
@@ -20,6 +20,7 @@
from .constants import *
import string
import secrets
+from django.conf import settings
from xkcdpass import xkcd_password as xp
@@ -47,6 +48,25 @@ def get_xyz_from_Gaussian_input(txt):
return xyz
+def get_xyz_from_cube(cube):
+ lines = cube.split("\n")
+ ind = 1
+ sline = lines[ind].strip().split()
+ xyz = ""
+ while len(sline) != 5:
+ ind += 1
+ sline = lines[ind].strip().split()
+ start_ind = ind
+
+ while len(sline) == 5:
+ xyz += f"{ATOMIC_SYMBOL[int(sline[0])]} {float(sline[2])*BOHR_VAL} {float(sline[3])*BOHR_VAL} {float(sline[4])*BOHR_VAL}\n"
+ ind += 1
+ sline = lines[ind].strip().split()
+
+ xyz = f"{ind-start_ind}\n\n{xyz}"
+ return xyz
+
+
def get_random_string(n=16):
return "".join(secrets.choice(full_alphabet) for i in range(n))
@@ -55,3 +75,62 @@ def get_random_readable_code(n=5):
wordfile = xp.locate_wordfile()
words = xp.generate_wordlist(wordfile=wordfile, min_length=n, max_length=n + 2)
return xp.generate_xkcdpassword(words)
+
+
+def job_triage(calc):
+ if calc.step.name in ["Conformational Search", "Constrained Conformational Search"]:
+ nproc = 8
+ elif calc.step.name == "Minimum Energy Path":
+ nproc = 4
+ else:
+ natoms = calc.structure.xyz_structure.count("\n") - 2
+ if calc.parameters.software == "xtb":
+ if natoms > 100:
+ nproc = 4
+ else:
+ nproc = 1
+ else:
+ if natoms < 20:
+ nproc = 4
+ else:
+ nproc = 8
+
+ user_type = calc.order.author.user_type
+
+ return (
+ min(settings.RESOURCE_LIMITS[user_type]["nproc"], nproc),
+ settings.RESOURCE_LIMITS[user_type]["time"] * 60,
+ )
+
+
+def guess_missing_parameters(params):
+ """
+ Guesses which software and driver to use based on the parameters.
+ Mostly for CalcUS Cloud at the moment.
+ """
+ if "calc_theory_level" in params:
+ t = params["calc_theory_level"]
+ if t == "xtb":
+ params["calc_software"] = "xtb"
+ params["calc_driver"] = "xtb"
+ if "calc_type" in params:
+ t = params["calc_type"]
+ if t == "TS Optimisation":
+ params["calc_driver"] = "Pysisyphus"
+ else:
+ params["calc_software"] = "NWChem"
+ params["calc_driver"] = "NWChem"
+ else:
+ params["calc_software"] = "xtb"
+ params["calc_driver"] = "xtb"
+
+ if "calc_solvent" in params:
+ solv = params["calc_solvent"]
+ if solv.lower() not in ["", "vacuum"]:
+ # There is a solvent
+ if params["calc_software"] == "xtb":
+ params["calc_solvation_model"] = "ALPB"
+ params["calc_solvation_radii"] = "default"
+ elif params["calc_software"] == "NWChem":
+ params["calc_solvation_model"] = "SMD"
+ params["calc_solvation_radii"] = "default"
diff --git a/frontend/management/commands/init_static_obj.py b/frontend/management/commands/init_static_obj.py
index deb59144..b4ef483a 100755
--- a/frontend/management/commands/init_static_obj.py
+++ b/frontend/management/commands/init_static_obj.py
@@ -58,17 +58,20 @@ def add_step(
self,
name,
short_name,
+ prop_name,
**kwargs,
):
if self.is_absent(BasicStep, name):
BasicStep.objects.create(
name=name,
short_name=short_name,
+ prop_name=prop_name,
**kwargs,
)
else:
BasicStep.objects.filter(name=name).update(
short_name=short_name,
+ prop_name=prop_name,
**kwargs,
)
@@ -78,100 +81,92 @@ def handle(self, *args, **options):
self.add_step(
"Geometrical Optimisation",
"opt",
+ "Geometry",
creates_ensemble=True,
- avail_xtb=True,
- avail_Gaussian=True,
- avail_ORCA=True,
)
self.add_step(
"Conformational Search",
"conf_search",
+ "Conformer ensemble",
creates_ensemble=True,
- avail_xtb=True,
- avail_Gaussian=False,
- avail_ORCA=False,
)
self.add_step(
"Constrained Optimisation",
"constr_opt",
+ "Geometry with constraint",
creates_ensemble=True,
- avail_xtb=True,
- avail_Gaussian=True,
- avail_ORCA=True,
)
self.add_step(
"Frequency Calculation",
"freq",
+ "Vibrational modes, IR spectrum and thermochemistry",
creates_ensemble=False,
- avail_xtb=True,
- avail_Gaussian=True,
- avail_ORCA=True,
)
self.add_step(
"TS Optimisation",
"optts",
+ "Transition state geometry",
creates_ensemble=True,
- avail_xtb=True,
- avail_Gaussian=True,
- avail_ORCA=True,
)
self.add_step(
"UV-Vis Calculation",
"uvvis",
+ "UV-Vis spectrum",
creates_ensemble=False,
- avail_xtb=True,
- avail_Gaussian=True,
- avail_ORCA=False,
)
self.add_step(
"NMR Prediction",
"nmr",
+ "NMR spectrum",
creates_ensemble=False,
- avail_xtb=False,
- avail_Gaussian=True,
- avail_ORCA=True,
)
self.add_step(
"Single-Point Energy",
"sp",
+ "Electronic energy",
creates_ensemble=False,
- avail_xtb=True,
- avail_Gaussian=True,
- avail_ORCA=True,
)
self.add_step(
"MO Calculation",
"mo",
+ "Molecular Orbitals",
creates_ensemble=False,
- avail_xtb=False,
- avail_Gaussian=False,
- avail_ORCA=True,
)
self.add_step(
"Minimum Energy Path",
"mep",
+ "Minimum Energy Path",
creates_ensemble=True,
- avail_xtb=True,
- avail_Gaussian=False,
- avail_ORCA=True,
)
self.add_step(
"Constrained Conformational Search",
"constr_conf_search",
+ "Conformer ensemble with constraint",
+ creates_ensemble=True,
+ )
+
+ self.add_step(
+ "ESP Calculation",
+ "esp",
+ "Electrostatic potential map",
+ creates_ensemble=False,
+ )
+
+ self.add_step(
+ "Fast Conformational Search",
+ "fast_conf_search",
+ "Preliminary conformer ensemble",
creates_ensemble=True,
- avail_xtb=True,
- avail_Gaussian=False,
- avail_ORCA=False,
)
title = "NHC-Catalysed Condensation"
diff --git a/frontend/management/commands/reschedule_calc.py b/frontend/management/commands/reschedule_calc.py
new file mode 100755
index 00000000..b0117dae
--- /dev/null
+++ b/frontend/management/commands/reschedule_calc.py
@@ -0,0 +1,34 @@
+import glob
+import os
+from django.core.management.base import BaseCommand
+
+from frontend.models import *
+from frontend.environment_variables import *
+from frontend.cloud_job import submit_cloud_job
+
+
+class Command(BaseCommand):
+ help = "Runs a calculation"
+
+ def add_arguments(self, parser):
+ parser.add_argument("calc_id", type=str)
+
+ def handle(self, *args, **options):
+ calc_id = options["calc_id"]
+ try:
+ calc = Calculation.objects.get(pk=calc_id)
+ except Calculation.DoesNotExist:
+ raise Exception(f"Could not find calculation number {calc_id}")
+
+ if calc.status == 2: # If successful
+ print(
+ f"Calculation {calc_id} has already completed successfully! This shouldn't happen"
+ )
+ return
+
+ submit_cloud_job(calc)
+
+ calc.status = 0
+ calc.save()
+
+ print(f"Calculation {calc_id} has been rescheduled")
diff --git a/frontend/models.py b/frontend/models.py
index 02514f97..d5023341 100644
--- a/frontend/models.py
+++ b/frontend/models.py
@@ -40,13 +40,14 @@
from hashid_field import HashidAutoField, BigHashidAutoField
from .constants import *
-from .helpers import get_random_readable_code
+from .helpers import get_random_readable_code, job_triage
+from .environment_variables import PAL
import ccinput
register = template.Library()
-STATUS_COLORS = {0: "#AAAAAA", 1: "#FFE515", 2: "#1EE000", 3: "#FD1425"}
+STATUS_COLORS = {0: "#202f26", 1: "#e2e100", 2: "#02b200", 3: "#b21b00"}
class UserManager(BaseUserManager):
@@ -119,6 +120,13 @@ class User(AbstractUser):
default_gaussian = models.CharField(max_length=1000, default="")
default_orca = models.CharField(max_length=1000, default="")
+ tour_done = models.BooleanField(default=False)
+ opted_in_emails = models.BooleanField(default=False)
+
+ advanced_interface = models.BooleanField(default=False)
+ calc_type_property = models.BooleanField(default=True)
+ calc_method_suggestions = models.BooleanField(default=True)
+
code = models.CharField(max_length=16) ### ?
pref_units = models.PositiveIntegerField(default=0)
@@ -130,6 +138,9 @@ class User(AbstractUser):
INV_UNITS = {v: k for k, v in UNITS.items()}
+ stripe_cus_id = models.CharField(max_length=256, default="")
+ stripe_will_renew = models.BooleanField(default=True)
+
# Total/allocated computation time and time consumed by user or class/group
# These numbers should be accurate, but are not the official references
# Instead, allocated_seconds should be recalculated from ResourceAllocation objects
@@ -149,6 +160,8 @@ def user_type(self):
elif self.is_subscriber:
return "subscriber"
else:
+ if self.member_of is not None and self.member_of.PI.is_subscriber:
+ return "subscriber"
return "free"
@property
@@ -268,14 +281,11 @@ class Subscription(models.Model):
"ResourceAllocation", on_delete=models.SET_NULL, null=True
)
- # Duration in months (1, 4, 12)
- # Mostly to identify which subscription option the user has chosen
- duration = models.PositiveSmallIntegerField()
- for_academia = models.BooleanField(default=True)
-
start_date = models.DateTimeField("date")
end_date = models.DateTimeField("date")
+ stripe_sub_id = models.CharField(max_length=256, default="")
+
class ResourceAllocation(models.Model):
id = BigHashidAutoField(
@@ -516,10 +526,12 @@ def connected(self):
class BasicStep(models.Model):
name = models.CharField(max_length=100)
short_name = models.CharField(max_length=100, default="")
+ prop_name = models.CharField(max_length=100, default="")
avail_xtb = models.BooleanField(default=False)
avail_Gaussian = models.BooleanField(default=False)
avail_ORCA = models.BooleanField(default=False)
+ avail_NWChem = models.BooleanField(default=False)
creates_ensemble = models.BooleanField(default=False)
@@ -881,6 +893,7 @@ class Property(models.Model):
uvvis = models.TextField(default="")
nmr = models.TextField(default="")
mo = models.TextField(default="")
+ esp = models.TextField(default="")
freq_list = ArrayField(models.FloatField(), default=list)
freq_animations = ArrayField(models.TextField(), default=list)
ir_spectrum = models.TextField(default="")
@@ -906,6 +919,10 @@ def has_nmr(self):
def has_mo(self):
return len(self.mo) > 0
+ @property
+ def has_esp(self):
+ return len(self.esp) > 0
+
class Structure(models.Model):
id = BigHashidAutoField(
@@ -1081,7 +1098,7 @@ class Molecule(models.Model):
primary_key=True, salt="Molecule_hashid_" + settings.HASHID_FIELD_SALT
)
name = models.CharField(max_length=100)
- inchi = models.CharField(max_length=1000, default="", blank=True, null=True)
+ inchi = models.CharField(max_length=10000, default="", blank=True, null=True)
project = models.ForeignKey(
Project, on_delete=models.CASCADE, blank=True, null=True
)
@@ -1249,6 +1266,12 @@ def see(self):
self.hidden = True
self.save()
+ @property
+ def total_cpu_time(self):
+ return sum(
+ [c.execution_time for c in self.calculation_set.filter(status__gt=0).all()]
+ )
+
@property
def color(self):
return STATUS_COLORS[self.status]
@@ -1502,17 +1525,30 @@ def delete(self, *args, **kwargs):
@property
def execution_time(self):
+ if self.date_started is None:
+ return 0
+ if self.date_finished is None:
+ end_date = timezone.now()
+ else:
+ end_date = self.date_finished
+
+ if settings.IS_CLOUD:
+ nproc, limit = job_triage(self)
+ delta = end_date - self.date_started
+ return round(delta.total_seconds() * nproc)
+ if self.local:
+ delta = end_date - self.date_started
+ return round(delta.total_seconds() * PAL)
+
if self.order.resource is None:
- return "-"
- if self.date_started is not None and self.date_finished is not None:
- if self.local:
- pal = os.getenv("OMP_NUM_THREADS")[0]
- else:
- pal = self.order.resource.pal
- return int((self.date_finished - self.date_started).seconds * int(pal))
+ # Shouldn't happen
+ return 0
+ if self.local:
+ pal = os.getenv("OMP_NUM_THREADS")[0]
else:
- return "-"
+ pal = self.order.resource.pal
+ return int((end_date - self.date_started).seconds * int(pal))
def __repr__(self):
return str(self.id)
@@ -1531,6 +1567,40 @@ def set_as_cancelled(self):
calc.status = 3
calc.save()
+
+class BatchCalcOrder(models.Model):
+ id = BigHashidAutoField(
+ primary_key=True, salt="BatchCalc_hashid_" + settings.HASHID_FIELD_SALT
+ )
+ step = models.ForeignKey(
+ BasicStep, on_delete=models.SET_NULL, blank=True, null=True
+ )
+ author = models.ForeignKey(User, on_delete=models.CASCADE, blank=True, null=True)
+
+ project = models.ForeignKey(
+ "Project", on_delete=models.CASCADE, blank=True, null=True
+ )
+ date = models.DateTimeField("date", null=True, blank=True)
+
+
+
+class BatchCalculation(models.Model):
+ id = BigHashidAutoField(
+ primary_key=True, salt="BatchCalc_hashid_" + settings.HASHID_FIELD_SALT
+ )
+ batch_name = models.CharField(max_length=100)
+ parametersets = models.JSONField(default=list)
+ parameters = models.ForeignKey(
+ Parameters, on_delete=models.SET_NULL, blank=True, null=True
+ )
+ structure = models.ForeignKey(
+ Structure, on_delete=models.SET_NULL, blank=True, null=True
+ )
+ calculationorder = models.ForeignKey(
+ BatchCalcOrder, n_delete=models.SET_NULL, blank=True, null=True)
+
+
+
class Filter(models.Model):
id = BigHashidAutoField(
diff --git a/frontend/tasks.py b/frontend/tasks.py
index 1a00ba77..18ab9c22 100644
--- a/frontend/tasks.py
+++ b/frontend/tasks.py
@@ -36,6 +36,7 @@
import shutil
import tempfile
import json
+import gzip, base64
from django.conf import settings
import threading
@@ -44,6 +45,13 @@
from shutil import copyfile, rmtree
import time
+from rdkit import Chem
+from rdkit.Chem import AllChem
+import spyrmsd
+from spyrmsd.optional.rdkit import to_molecule
+from spyrmsd.rmsd import rmsdwrapper
+from sklearn.cluster import DBSCAN
+
if not settings.IS_CLOUD:
from calcus.celery import app
from celery.signals import task_prerun, task_postrun
@@ -75,6 +83,7 @@ def wrapper(*args, **kwargs):
from django.utils import timezone
from django.core import management
+from django.core.mail import send_mail
from django.db.utils import IntegrityError
from ccinput.wrapper import generate_calculation
@@ -92,10 +101,10 @@ def wrapper(*args, **kwargs):
import logging
+logger = logging.getLogger()
logging.basicConfig(
level=logging.INFO, format="%(asctime)s [%(levelname)s] %(module)s: %(message)s"
)
-logger = logging.getLogger(__name__)
REMOTE = False
@@ -335,7 +344,14 @@ def testing_delay_cloud(calc_id):
return ErrorCodes.SUCCESS
-def system(command, log_file="", force_local=False, software="xtb", calc_id=-1):
+def system(
+ command,
+ log_file="",
+ force_local=False,
+ software="xtb",
+ calc_id=-1,
+ time_cumul=False,
+):
if REMOTE and not force_local:
assert calc_id != -1
@@ -449,16 +465,20 @@ def system(command, log_file="", force_local=False, software="xtb", calc_id=-1):
return ret
else: # Local
+ # Assume the calculation will be run in the same directory as the output file
+ exe_dir = os.path.dirname(log_file)
+ logger.info(f"exe_dir: {exe_dir} (calc {calc_id})")
if calc_id != -1 and settings.IS_CLOUD:
try:
- timeout = int(os.getenv("CALCUS_TIMEOUT", "300"))
+ timeout = int(os.getenv("CALCUS_TIMEOUT", "580"))
except ValueError:
logger.warning(f"Could not parse the timeout for calc {calc_id}")
- timeout = 300
+ timeout = 580
if calc_id != -1:
calc = Calculation.objects.get(pk=calc_id)
calc.status = 1
- calc.date_started = timezone.now()
+ if calc.date_started is None or not time_cumul:
+ calc.date_started = timezone.now()
calc.save()
res = AbortableAsyncResult(calc.task_id)
@@ -471,7 +491,13 @@ def system(command, log_file="", force_local=False, software="xtb", calc_id=-1):
return testing_delay_local(res)
try:
- t = subprocess.Popen(shlex.split(command), stdout=stream, stderr=stream)
+ if exe_dir.strip() != "":
+ t = subprocess.Popen(
+ shlex.split(command), stdout=stream, stderr=stream, cwd=exe_dir
+ )
+ else:
+ t = subprocess.Popen(shlex.split(command), stdout=stream, stderr=stream)
+
except FileNotFoundError:
logger.error(f'Could not run command "{command}" - executable not found')
calc.error_message = f"{command.split()[0]} is not found"
@@ -507,6 +533,10 @@ def kill_task():
return ErrorCodes.SUCCESS
else:
logger.info(f"Got returncode {t.returncode}")
+ with open(log_file) as f:
+ last_lines = "".join(f.readlines()[-100:])
+ logger.info("Last lines of the log file: " + last_lines)
+
return ErrorCodes.UNKNOWN_TERMINATION
if calc_id != -1:
@@ -602,7 +632,7 @@ def setup_cached_calc(calc):
return True
-def generate_xyz_structure(drawing, inp, ext):
+def generate_xyz_structure(drawing, inp, ext, scale=1):
if ext == "xyz":
return inp
elif ext == "mol":
@@ -621,15 +651,19 @@ def generate_xyz_structure(drawing, inp, ext):
to_print = []
for line in inp.split("\n")[4:]:
sline = line.split()
+ if len(sline) < 4:
+ # Not sure what input causes this error
+ logger.warning(f"Strange .mol input; processing line: '{line}'")
+ break
try:
a = int(sline[3])
except ValueError:
to_print.append(
"{} {} {} {}\n".format(
sline[3],
- float(sline[0]),
- float(sline[1]),
- float(sline[2]),
+ float(sline[0]) * scale,
+ float(sline[1]) * scale,
+ float(sline[2]) * scale,
)
)
else:
@@ -677,9 +711,8 @@ def launch_pysis_calc(in_file, calc, files):
with open(os.path.join(local_folder, "calc.inp"), "w") as out:
out.write(calc.input_file)
- os.chdir(local_folder)
-
if not calc.local:
+ log_path = os.path.join(folder, "calc.out")
pid = int(threading.get_ident())
conn = connections[pid]
lock = locks[pid]
@@ -700,9 +733,9 @@ def launch_pysis_calc(in_file, calc, files):
lock,
)
else:
- os.chdir(local_folder)
+ log_path = os.path.join(local_folder, "calc.out")
- ret = system(calc.command, "calc.out", software="pysis", calc_id=calc.id)
+ ret = system(calc.command, log_path, software="pysis", calc_id=calc.id)
cancelled = False
if ret != ErrorCodes.SUCCESS:
@@ -795,6 +828,7 @@ def launch_xtb_calc(in_file, calc, files):
os.chdir(local_folder)
if not calc.local:
+ log_path = os.path.join(folder, local_folder)
pid = int(threading.get_ident())
conn = connections[pid]
lock = locks[pid]
@@ -815,9 +849,9 @@ def launch_xtb_calc(in_file, calc, files):
lock,
)
else:
- os.chdir(local_folder)
+ log_path = os.path.join(local_folder, "calc.out")
- ret = system(calc.command, "calc.out", software="xtb", calc_id=calc.id)
+ ret = system(calc.command, log_path, software="xtb", calc_id=calc.id)
cancelled = False
if ret != ErrorCodes.SUCCESS:
@@ -1373,7 +1407,6 @@ def xtb_freq(in_file, calc):
lines = [i + "\n" for i in calc.structure.xyz_structure.split("\n")]
num_atoms = int(lines[0].strip())
lines = lines[2:]
- hess = []
struct = []
for line in lines:
@@ -1507,6 +1540,135 @@ def crest(in_file, calc):
return ErrorCodes.SUCCESS
+def fast_conf(in_file, calc):
+ local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id))
+ folder = f"scratch/calcus/{calc.id}" # Remote
+
+ calc_start = timezone.now()
+
+ subprocess.call(shlex.split("obabel in.xyz -O in.mol"), cwd=local_folder)
+
+ mol = Chem.MolFromMolFile(os.path.join(local_folder, "in.mol"))
+
+ mol.UpdatePropertyCache()
+ mol = Chem.AddHs(mol)
+
+ confs = list(
+ AllChem.EmbedMultipleConfs(
+ mol, numConfs=50, maxAttempts=1000, pruneRmsThresh=0.05
+ )
+ )
+ mols = []
+ props = []
+
+ for num in confs:
+ cpath = os.path.join(local_folder, f"conf{num+1}")
+ os.makedirs(cpath, exist_ok=True)
+
+ Chem.rdmolfiles.MolToXYZFile(mol, os.path.join(cpath, "in.xyz"), confId=num)
+
+ os.chdir(cpath)
+
+ # We want to add up all the time required for the calculations.
+ # We will overwrite this with a more accurate time afterwards, but this makes the time preview
+ # a bit less strange.
+ ret = system(
+ calc.command,
+ os.path.join(cpath, "calc.out"),
+ software="xtb",
+ calc_id=calc.id,
+ time_cumul=True,
+ )
+
+ # duplicate code
+ if ret != ErrorCodes.SUCCESS:
+ logger.error(f"Failed to optimize conformer {num} of calc {calc.id}")
+ continue
+
+ with open(f"{cpath}/xtbopt.xyz") as f:
+ xyz = clean_xyz("".join(f.readlines()))
+
+ with open(f"{cpath}/calc.out") as f:
+ lines = f.readlines()
+ ind = len(lines) - 1
+
+ try:
+ while lines[ind].find("TOTAL ENERGY") == -1:
+ ind -= 1
+ E = float(lines[ind].split()[3])
+ except IndexError:
+ logger.error(
+ f"Could not parse the output of calculation {calc.id}: invalid format"
+ )
+ continue
+
+ rdmol = Chem.MolFromMolFile(os.path.join(cpath, "xtbtopo.mol"))
+
+ smol = to_molecule(rdmol)
+
+ mols.append(smol)
+ props.append((E, xyz))
+
+ def rmsd_cluster(mols, props, threshold=0.8):
+ # Could be too slow if there are many structures
+ dist = np.zeros((len(mols), len(mols)))
+ for i, (mol1, prop1) in enumerate(zip(mols, props)):
+ for j, (mol2, prop2) in enumerate(zip(mols, props)):
+ if j <= i:
+ continue
+ try:
+ d = rmsdwrapper(
+ mol1, mol2, symmetry=True, center=True, minimize=True
+ )[0]
+ except spyrmsd.exceptions.NonIsomorphicGraphs:
+ d = 10
+ logger.error(f"Non-isomorphic graphs: {i} and {j} (calc {calc.id})")
+
+ dist[i, j] = d
+ dist[j, i] = d
+
+ clustering = DBSCAN(eps=threshold, min_samples=1, metric="precomputed").fit(
+ dist
+ )
+
+ unique_mol = []
+ unique_prop = []
+ known = [] # Known cluster labels
+ for ind, pos in enumerate(clustering.labels_):
+ if pos in known:
+ continue
+ known.append(pos)
+ unique_mol.append(mols[ind])
+ unique_prop.append(props[ind])
+
+ return unique_mol, unique_prop
+
+ umols, uprops = rmsd_cluster(mols, props, threshold=0.5)
+ for ind, (mol, (E, xyz)) in enumerate(
+ sorted(zip(umols, uprops), key=lambda i: i[1])
+ ):
+ s = Structure.objects.get_or_create(
+ parent_ensemble=calc.result_ensemble,
+ number=ind + 1,
+ )[0]
+ s.degeneracy = 1
+ s.xyz_structure = xyz
+ prop = get_or_create(calc.parameters, s)
+ prop.energy = E
+ prop.geom = True
+ s.save()
+ prop.save()
+
+ calc_end = timezone.now()
+
+ # Set the true start and end dates, including the conformational sampling and parsing
+ calc.date_started = calc_start
+ calc.date_finished = calc_end
+ calc.save()
+
+ return ErrorCodes.SUCCESS
+
+
def clean_struct_line(line):
a, x, y, z = line.split()
return f"{LOWERCASE_ATOMIC_SYMBOLS[a.lower()]} {x} {y} {z}\n"
@@ -1546,10 +1708,9 @@ def launch_orca_calc(in_file, calc, files):
"$EBROOTORCA/orca calc.inp", "calc.out", software="ORCA", calc_id=calc.id
)
else:
- os.chdir(local_folder)
ret = system(
f"{EBROOTORCA}/orca calc.inp",
- "calc.out",
+ os.path.join(local_folder, "calc.out"),
software="ORCA",
calc_id=calc.id,
)
@@ -1627,7 +1788,10 @@ def orca_mo_gen(in_file, calc):
return ErrorCodes.MISSING_FILE
with open(path) as f:
cube = "".join(f.readlines())
- cubes[mo] = cube
+
+ comp_cube = gzip.compress(bytes(cube, "utf-8"))
+ str_cube = base64.b64encode(comp_cube).decode("utf-8")
+ cubes[mo] = str_cube
prop.energy = E
prop.mo = json.dumps(cubes)
@@ -2204,6 +2368,492 @@ def xtb_stda(in_file, calc): # TO OPTIMIZE
return ErrorCodes.SUCCESS
+def launch_nwchem_calc(in_file, calc, files):
+ local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id))
+ folder = f"scratch/calcus/{calc.id}"
+
+ if not os.path.isdir(local_folder):
+ os.makedirs(local_folder, exist_ok=True)
+
+ # Debugging code for now
+ logger.info(f"The input is {calc.input_file}")
+
+ with open(os.path.join(local_folder, "calc.inp"), "w") as out:
+ out.write(calc.input_file)
+
+ if not calc.local:
+ pid = int(threading.get_ident())
+ conn = connections[pid]
+ lock = locks[pid]
+ remote_dir = remote_dirs[pid]
+ if calc.remote_id == 0:
+ sftp_put(
+ f"{local_folder}/calc.inp",
+ os.path.join(folder, "calc.inp"),
+ conn,
+ lock,
+ )
+
+ ret = system(
+ "mpirun -n {PAL} nwchem calc.inp",
+ "calc.out",
+ software="NWChem",
+ calc_id=calc.id,
+ )
+ else:
+ if settings.IS_CLOUD:
+ cmd = f"mpirun -n {PAL} --use-hwthread-cpus nwchem calc.inp"
+ else:
+ cmd = f"mpirun -n {PAL} nwchem calc.inp"
+
+ ret = system(
+ cmd,
+ os.path.join(local_folder, "calc.out"),
+ software="NWChem",
+ calc_id=calc.id,
+ )
+
+ cancelled = False
+ if ret != ErrorCodes.SUCCESS:
+ if ret == ErrorCodes.JOB_CANCELLED:
+ cancelled = True
+ else:
+ return ret
+
+ if not calc.local:
+ for f in files:
+ a = sftp_get(
+ f"{folder}/{f}",
+ os.path.join(CALCUS_SCR_HOME, str(calc.id), f),
+ conn,
+ lock,
+ )
+ if not cancelled and a != ErrorCodes.SUCCESS:
+ return a
+
+ if not cancelled:
+ for f in files:
+ if not os.path.isfile(f"{local_folder}/{f}"):
+ return ErrorCodes.COULD_NOT_GET_REMOTE_FILE
+
+ return ErrorCodes.SUCCESS
+ else:
+ return ErrorCodes.JOB_CANCELLED
+
+
+def nwchem_sp(in_file, calc):
+ local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id))
+
+ ret = launch_nwchem_calc(in_file, calc, ["calc.out"])
+
+ if ret != ErrorCodes.SUCCESS:
+ return ret
+
+ with open(f"{local_folder}/calc.out") as f:
+ lines = f.readlines()
+ ind = len(lines) - 1
+
+ try:
+ while (
+ lines[ind].find("Total SCF energy") == -1
+ and lines[ind].find("Total DFT energy") == -1
+ ):
+ ind -= 1
+ E = float(lines[ind].split()[-1])
+ except IndexError:
+ logger.error(
+ f"Could not parse the output of calculation {calc.id}: invalid format"
+ )
+ return ErrorCodes.INVALID_OUTPUT
+
+ prop = get_or_create(calc.parameters, calc.structure)
+ prop.energy = E
+ prop.save()
+
+ # parse_nwchem_charges(calc, calc.structure)
+
+ return ErrorCodes.SUCCESS
+
+
+def nwchem_mo_gen(in_file, calc):
+ local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id))
+
+ ret = launch_nwchem_calc(
+ in_file,
+ calc,
+ ["calc.out", "in-HOMO.cube", "in-LUMO.cube", "in-LUMOA.cube", "in-LUMOB.cube"],
+ )
+
+ if ret != ErrorCodes.SUCCESS:
+ return ret
+
+ with open(f"{local_folder}/calc.out") as f:
+ lines = f.readlines()
+ ind = len(lines) - 1
+
+ try:
+ while (
+ lines[ind].find("Total SCF energy") == -1
+ and lines[ind].find("Total DFT energy") == -1
+ ):
+ ind -= 1
+ E = float(lines[ind].split()[-1])
+ except IndexError:
+ logger.error(
+ f"Could not parse the output of calculation {calc.id}: invalid format"
+ )
+ return ErrorCodes.INVALID_OUTPUT
+
+ prop = get_or_create(calc.parameters, calc.structure)
+
+ cubes = {}
+ for mo in ["HOMO", "LUMO", "LUMOA", "LUMOB"]:
+ path = os.path.join(local_folder, f"in-{mo}.cube")
+ if not os.path.isfile(path):
+ logger.error(f"Cube file {path} does not exist!")
+ return ErrorCodes.MISSING_FILE
+ with open(path) as f:
+ cube = "".join(f.readlines())
+
+ comp_cube = gzip.compress(bytes(cube, "utf-8"))
+ str_cube = base64.b64encode(comp_cube).decode("utf-8")
+ cubes[mo] = str_cube
+
+ prop.energy = E
+ prop.mo = json.dumps(cubes)
+ prop.save()
+
+ # parse_nwchem_charges(calc, calc.structure)
+
+ return ErrorCodes.SUCCESS
+
+
+def nwchem_esp_gen(in_file, calc):
+ local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id))
+
+ ret = launch_nwchem_calc(
+ in_file,
+ calc,
+ ["calc.out", "esp.cube", "in.eld.total.cube"],
+ )
+
+ if ret != ErrorCodes.SUCCESS:
+ return ret
+
+ # TODO: refactor this?
+ with open(f"{local_folder}/calc.out") as f:
+ lines = f.readlines()
+ ind = len(lines) - 1
+
+ try:
+ while (
+ lines[ind].find("Total SCF energy") == -1
+ and lines[ind].find("Total DFT energy") == -1
+ ):
+ ind -= 1
+ E = float(lines[ind].split()[-1])
+ except IndexError:
+ logger.error(
+ f"Could not parse the output of calculation {calc.id}: invalid format"
+ )
+ return ErrorCodes.INVALID_OUTPUT
+
+ prop = get_or_create(calc.parameters, calc.structure)
+
+ cubes = {}
+ MO_LABELS = {
+ "esp": "esp",
+ "in.eld.total": "density",
+ }
+ for mo in ["esp", "in.eld.total"]:
+ path = os.path.join(local_folder, f"{mo}.cube")
+ if not os.path.isfile(path):
+ logger.error(f"Cube file {path} does not exist!")
+ return ErrorCodes.MISSING_FILE
+ with open(path) as f:
+ cube = "".join(f.readlines())
+
+ # both could be compressed together, I suppose
+ comp_cube = gzip.compress(bytes(cube, "utf-8"))
+ str_cube = base64.b64encode(comp_cube).decode("utf-8")
+ cubes[MO_LABELS[mo]] = str_cube
+
+ prop.energy = E
+ prop.esp = json.dumps(cubes)
+ prop.save()
+
+ # parse_nwchem_charges(calc, calc.structure)
+
+ return ErrorCodes.SUCCESS
+
+
+def nwchem_opt(in_file, calc):
+ local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id))
+
+ ret = launch_nwchem_calc(in_file, calc, ["calc.out"])
+
+ if ret != ErrorCodes.SUCCESS:
+ return ret
+
+ converged = True
+ structures = []
+ rmsds = []
+ with open(f"{local_folder}/calc.out") as f:
+ lines = f.readlines()
+ ind = 0
+ while True:
+ while (
+ ind < len(lines) - 1
+ and lines[ind].find('Geometry "geometry" -> "geometry"') == -1
+ ):
+ ind += 1
+ if ind >= len(lines) - 1:
+ break
+ ind += 7
+ xyz = []
+
+ while lines[ind].strip() != "":
+ sline = lines[ind].split()
+ ind += 1
+ xyz.append([sline[1], [float(i) for i in sline[3:]]])
+
+ structures.append(xyz)
+ while (
+ lines[ind].find(
+ "Step Energy Delta E Gmax Grms Xrms Xmax Walltime"
+ )
+ == -1
+ and ind < len(lines) - 1
+ ):
+ ind += 1
+
+ if ind >= len(lines) - 1:
+ break
+
+ ind += 2
+ RMSD = float(lines[ind].split()[6])
+ rmsds.append(RMSD)
+
+ try:
+ while lines[ind].find("Optimization converged") == -1:
+ ind -= 1
+ except IndexError:
+ logger.warning(f"Optimization {calc.id} did not converge")
+ else:
+ converged = True
+ try:
+ while (
+ lines[ind].find("Total SCF energy") == -1
+ and lines[ind].find("Total DFT energy") == -1
+ ):
+ ind -= 1
+ E = float(lines[ind].split()[-1])
+ except IndexError:
+ return ErrorCodes.INVALID_OUTPUT
+
+ if converged:
+ s = Structure.objects.get_or_create(
+ parent_ensemble=calc.result_ensemble,
+ number=calc.structure.number,
+ )[0]
+ s.degeneracy = calc.structure.degeneracy
+ s.xyz_structure = format_xyz(structures[-1])
+
+ prop = get_or_create(calc.parameters, s)
+ prop.energy = E
+ prop.geom = True
+ s.save()
+ prop.save()
+
+ frames = structures[:-1]
+ else:
+ frames = structures
+
+ new_frames = []
+ update_frames = []
+
+ for ind, (s, rmsd) in enumerate(zip(frames, rmsds)):
+ xyz = format_xyz(s)
+ try:
+ f = calc.calculationframe_set.get(number=ind + 1)
+ except CalculationFrame.DoesNotExist:
+ new_frames.append(
+ CalculationFrame(
+ number=ind + 1,
+ xyz_structure=xyz,
+ parent_calculation=calc,
+ RMSD=rmsd,
+ )
+ )
+ else:
+ f.xyz_structure = xyz
+ f.RMSD = rmsd
+ update_frames.append(f)
+
+ CalculationFrame.objects.bulk_create(new_frames)
+ CalculationFrame.objects.bulk_update(update_frames, ["xyz_structure", "RMSD"])
+
+ # parse_nwchem_charges(calc, s)
+
+ return ErrorCodes.SUCCESS
+
+
+def nwchem_freq(in_file, calc):
+ local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id))
+
+ ret = launch_nwchem_calc(in_file, calc, ["calc.out"])
+
+ if ret != ErrorCodes.SUCCESS:
+ return ret
+
+ base_xyz = [i + "\n" for i in calc.structure.xyz_structure.split("\n")]
+ num_atoms = int(base_xyz[0].strip())
+ base_xyz = base_xyz[2:]
+ struct = []
+ prop = get_or_create(calc.parameters, calc.structure)
+
+ for line in base_xyz:
+ if line.strip() != "":
+ a, x, y, z = line.strip().split()
+ struct.append([a, float(x), float(y), float(z)])
+
+ with open(f"{local_folder}/calc.out") as f:
+ lines = f.readlines()
+ ind = len(lines) - 1
+
+ try:
+ while lines[ind].find("Zero-Point correction to Energy") == -1:
+ ind -= 1
+ # Temperature = 298.15K
+ T = float(lines[ind - 3].split()[-1][:1]) # K
+
+ ZPE = float(lines[ind].split()[-2]) # Eh
+ Hcorr = float(lines[ind + 2].split()[-2]) # Eh
+ S = float(lines[ind + 4].split()[-2]) # cal/(mol*K)
+
+ while (
+ lines[ind].find("Total SCF energy") == -1
+ and lines[ind].find("Total DFT energy") == -1
+ ):
+ ind -= 1
+ E = float(lines[ind].split()[-1]) # Eh
+ except IndexError:
+ raise
+ logger.error(
+ f"Could not parse the output of calculation {calc.id}: invalid format"
+ )
+ return ErrorCodes.INVALID_OUTPUT
+
+ G = E + Hcorr - T * S / (1000 * HARTREE_TO_KCAL_F)
+
+ prop.energy = E
+ prop.free_energy = G
+
+ try:
+ while (
+ lines[ind].find("NORMAL MODE EIGENVECTORS IN CARTESIAN COORDINATES")
+ == -1
+ ):
+ ind += 1
+
+ ind += 4
+
+ vib_freqs = []
+ freqs = [] # each entry is the animation for one vib mode
+ while lines[ind].strip() != "":
+ num_line = len(lines[ind].strip().split())
+ end_num = int(lines[ind].split()[-1])
+ ind += 2
+
+ vib_freqs += [float(i) for i in lines[ind].split()[1:]]
+
+ ind += 2
+
+ vib = []
+ for i in range(num_line):
+ vib.append([])
+
+ for j in range(num_atoms * 3):
+ sline = lines[ind].split()
+ for i in range(num_line):
+ coord = float(sline[1 + i])
+ vib[i].append(coord)
+ ind += 1
+
+ ####
+ def is_all_null(arr):
+ for el in arr:
+ if float(el) != 0:
+ return False
+ return True
+
+ freqs += vib
+ ind += 1
+ except IndexError:
+ raise
+ logger.error(
+ f"Could not parse the frequency modes of calculation {calc.id}: invalid format"
+ )
+ return ErrorCodes.INVALID_OUTPUT
+
+ freq_animations = []
+ for ind in range(len(freqs)):
+ anim = f"{num_atoms}\nCalcUS\n"
+ assert len(struct) == num_atoms
+ for ind2, (a, x, y, z) in enumerate(struct):
+ anim += "{} {:.4f} {:.4f} {:.4f} {} {} {}\n".format(
+ a, x, y, z, *freqs[ind][3 * ind2 : 3 * ind2 + 3]
+ )
+ freq_animations.append(anim)
+
+ # Remove translation and rotation
+ # TODO: test for linear molecules
+ prop.freq_animations = freq_animations[6:]
+ prop.freq_list = vib_freqs[6:]
+ prop.save()
+
+ # vib_file = os.path.join(local_folder, "vibspectrum")
+
+ """
+ if os.path.isfile(vib_file):
+ with open(vib_file) as f:
+ lines = f.readlines()
+
+ vibs = []
+ intensities = []
+ for line in lines:
+ if len(line.split()) > 4 and line[0] != "#":
+ sline = line.split()
+ try:
+ a = float(sline[1])
+ if a == 0.0:
+ continue
+ except ValueError:
+ pass
+ vib = float(line[20:33].strip())
+ vibs.append(vib)
+ try:
+ intensity = float(sline[3])
+ except ValueError:
+ continue
+ else:
+ intensities.append(intensity)
+ if len(vibs) == len(intensities) and len(intensities) > 0:
+ x = np.arange(500, 4000, 1) # Wave number in cm^-1
+ spectrum = plot_vibs(x, zip(vibs, intensities))
+ data = "Wavenumber,Intensity\n"
+ intensities = 1000 * np.array(intensities) / max(intensities)
+ for _x, i in sorted((zip(list(x), spectrum)), reverse=True):
+ data += f"-{_x:.1f},{i:.5f}\n"
+
+ prop.ir_spectrum = data
+ prop.freq_list = vibs
+
+ """
+ prop.save()
+ return ErrorCodes.SUCCESS
+
+
def calc_to_ccinput(calc):
if calc.parameters.method != "":
_method = calc.parameters.method
@@ -2215,10 +2865,11 @@ def calc_to_ccinput(calc):
raise Exception(
f"No method specified; theory level is {calc.parameters.theory_level}"
)
+
_specifications = calc.parameters.specifications
software = calc.parameters.software.lower()
- if software in ["gaussian", "orca"]:
+ if software in ["gaussian", "orca"]: # TODO: NWChem?
_specifications += " " + getattr(calc.order.author, "default_" + software)
_solvation_radii = calc.parameters.solvation_radii
else:
@@ -2240,9 +2891,30 @@ def calc_to_ccinput(calc):
_nproc = calc.order.resource.pal
_mem = calc.order.resource.memory
+ _step_name = calc.step.name
+
+ if calc.parameters.software.lower() == "xtb" and (
+ calc.step.name == "Fast Conformational Search"
+ ):
+ _step_name = "OPT"
+
+ if (
+ calc.parameters.software.lower() == "xtb"
+ and (calc.parameters.driver.lower() == "pysis")
+ and (calc.step.name == "TS Optimization")
+ ):
+ if "tsopt(thresh=" not in _specifications:
+ _specifications += "tsopt(thresh=gau_vtight)"
+
+ # patch
+ if calc.parameters.software.lower() == "nwchem" and (
+ calc.step.name == "MO Calculation" or calc.step.name == "ESP Calculation"
+ ):
+ _step_name = "SP"
+
params = {
"software": calc.parameters.software,
- "type": calc.step.name,
+ "type": _step_name,
"method": _method,
"basis_set": calc.parameters.basis_set,
"solvent": calc.parameters.solvent,
@@ -2261,9 +2933,13 @@ def calc_to_ccinput(calc):
"name": "in",
"driver": calc.parameters.driver,
}
+
try:
inp = generate_calculation(**params)
- except CCInputException as e:
+ except Exception as e:
+ logger.error(
+ f"Could not generate the input file: got exception {str(e)} from ccinput"
+ )
return e
return inp
@@ -2295,6 +2971,7 @@ def launch_gaussian_calc(in_file, calc, files):
ret = system("g16 calc.com", software="Gaussian", calc_id=calc.id)
else:
os.chdir(local_folder)
+ # There could be a race condition here
ret = system("g16 calc.com", software="Gaussian", calc_id=calc.id)
cancelled = False
@@ -3236,12 +3913,14 @@ def gen_fingerprint(structure):
stderr=stream,
).decode("utf-8")
- inchi = out.split("\n")[-2]
- if inchi[:6] != "InChI=":
+ stext = out.split("\n")
+
+ for line in stext:
+ if line[:6] == "InChI=":
+ return line[6:]
+ else:
logger.warning(f"Invalid InChI key obtained for structure {structure.id}")
return ""
- else:
- return inchi[6:]
def analyse_opt(calc_id):
@@ -3251,6 +3930,7 @@ def analyse_opt(calc_id):
"xtb": analyse_opt_xtb,
"pysis": analyse_opt_pysis,
"pysisyphus": analyse_opt_pysis,
+ "nwchem": analyse_opt_nwchem,
}
calc = Calculation.objects.get(pk=calc_id)
@@ -3266,7 +3946,16 @@ def analyse_opt(calc_id):
return funcs[calc.parameters.software.lower()](calc)
+def analyse_opt_nwchem(calc):
+ # TODO
+ return
+
+
def analyse_opt_pysis(calc):
+ logger.warning(
+ "Parsing the Pysisyphus optimisation trajectory is currently not implemented"
+ )
+ return
# TODO
prepath = os.path.join(CALCUS_SCR_HOME, str(calc.id))
@@ -3565,6 +4254,7 @@ def get_Gaussian_xyz(text):
SPECIAL_FUNCTIONALS = ["HF-3c", "PBEh-3c"]
+
BASICSTEP_TABLE = {
"xtb": {
"Geometrical Optimisation": xtb_opt,
@@ -3576,6 +4266,7 @@ def get_Gaussian_xyz(text):
"Single-Point Energy": xtb_sp,
"Minimum Energy Path": mep,
"Constrained Conformational Search": crest,
+ "Fast Conformational Search": fast_conf,
},
"ORCA": {
"NMR Prediction": orca_nmr,
@@ -3596,6 +4287,17 @@ def get_Gaussian_xyz(text):
"Single-Point Energy": gaussian_sp,
"UV-Vis Calculation": gaussian_td,
},
+ "NWChem": {
+ # "NMR Prediction": nwchem_nmr,
+ "Geometrical Optimisation": nwchem_opt,
+ # "TS Optimisation": nwchem_ts, ##############
+ "MO Calculation": nwchem_mo_gen,
+ "ESP Calculation": nwchem_esp_gen,
+ "Frequency Calculation": nwchem_freq,
+ # "Constrained Optimisation": nwchem_scan,
+ "Single-Point Energy": nwchem_sp,
+ # "Minimum Energy Path": mep,
+ },
}
time_dict = {}
@@ -3733,6 +4435,10 @@ def dispatcher(order_id, drawing=None, is_flowchart=False, flowchartStepObjectId
)
except Molecule.DoesNotExist:
"""
+
+ if len(fingerprint) > 9900:
+ fingerprint = fingerprint[:9900]
+
ensemble.parent_molecule.inchi = fingerprint
ensemble.parent_molecule.save()
molecule = ensemble.parent_molecule
@@ -3913,9 +4619,79 @@ def dispatcher(order_id, drawing=None, is_flowchart=False, flowchartStepObjectId
c.save()
+NWCHEM_MO_FIX = """
+dplot
+ TITLE HOMO
+ vectors in.movecs
+ LimitXYZ
+ -5.0 5.0 40
+ -5.0 5.0 40
+ -5.0 5.0 40
+ spin total
+ ORBITALS view; 1; {}
+ gaussian
+ output in-HOMO.cube
+end
+task dplot
+
+dplot
+ TITLE LUMO
+ vectors in.movecs
+ LimitXYZ
+ -5.0 5.0 40
+ -5.0 5.0 40
+ -5.0 5.0 40
+ spin total
+ ORBITALS view; 1; {}
+ gaussian
+ output in-LUMO.cube
+end
+task dplot
+
+dplot
+ TITLE LUMOA
+ vectors in.movecs
+ LimitXYZ
+ -5.0 5.0 40
+ -5.0 5.0 40
+ -5.0 5.0 40
+ spin total
+ ORBITALS view; 1; {}
+ gaussian
+ output in-LUMOA.cube
+end
+task dplot
+
+dplot
+ TITLE LUMOB
+ vectors in.movecs
+ LimitXYZ
+ -5.0 5.0 40
+ -5.0 5.0 40
+ -5.0 5.0 40
+ spin total
+ ORBITALS view; 1; {}
+ gaussian
+ output in-LUMOB.cube
+end
+task dplot
+
+"""
+
+NWCHEM_ESP_FIX = """
+property
+ ESP
+ ELECTRONDENSITY
+ grid pad 2.0 step 0.2 output esp.cube
+end
+
+task dft property
+"""
+
+
def add_input_to_calc(calc):
inp = calc_to_ccinput(calc)
- if isinstance(inp, CCInputException):
+ if isinstance(inp, Exception):
msg = f"CCInput error: {str(inp)}"
if IS_TEST or settings.DEBUG:
print(msg)
@@ -3924,10 +4700,35 @@ def add_input_to_calc(calc):
calc.save()
return ErrorCodes.FAILED_TO_CREATE_INPUT
- calc.input_file = inp.input_file
+ input_file = inp.input_file
+
+ # Quick fix
+ if (
+ calc.parameters.software.lower() == "nwchem"
+ and calc.step.name == "MO Calculation"
+ ):
+ electrons = 0
+ for line in calc.structure.xyz_structure.split("\n")[2:]:
+ if line.strip() == "":
+ continue
+ el = line.split()[0]
+ electrons += ATOMIC_NUMBER[el]
+
+ electrons -= calc.parameters.charge
+ n_homo = electrons // 2
+
+ input_file += NWCHEM_MO_FIX.format(n_homo, n_homo + 1, n_homo + 2, n_homo + 3)
+ elif (
+ calc.parameters.software.lower() == "nwchem"
+ and calc.step.name == "ESP Calculation"
+ ):
+ input_file += NWCHEM_ESP_FIX
+
+ calc.input_file = input_file
- if hasattr(inp, "confirmed_specifications"):
- calc.parameters.specifications = inp.confirmed_specifications
+ # TODO: make ccinput support keywords in a more general way
+ # if hasattr(inp, "confirmed_specifications"):
+ # calc.parameters.specifications = inp.confirmed_specifications
if hasattr(inp, "command"):
calc.command = inp.command
@@ -3937,7 +4738,16 @@ def add_input_to_calc(calc):
def load_output_files(calc):
- output_files = {}
+ if not os.path.isdir(os.path.join(CALCUS_SCR_HOME, str(calc.id))):
+ logger.warning(
+ f"Cannot load the logs of calculation {calc.id}: no scratch directory"
+ )
+ return
+
+ if calc.output_files.strip() in ["{}", ""]:
+ output_files = {}
+ else:
+ output_files = json.loads(calc.output_files)
for log_name in glob.glob(
os.path.join(CALCUS_SCR_HOME, str(calc.id), "*.log")
) + glob.glob(os.path.join(CALCUS_SCR_HOME, str(calc.id), "*.out")):
@@ -3946,6 +4756,10 @@ def load_output_files(calc):
log = "".join(f.readlines())
output_files[fname] = log
+ logger.info(
+ f"Loaded {len(output_files)} output file(s) for calculation {str(calc.id)} (scr: {CALCUS_SCR_HOME})"
+ )
+
calc.output_files = json.dumps(output_files)
calc.save()
@@ -3990,6 +4804,7 @@ def get_calc(calc_id):
ret = None
else:
ret = add_input_to_calc(calc)
+ calc.refresh_from_db()
if isinstance(ret, ErrorCodes):
return ret
@@ -4049,8 +4864,8 @@ def get_calc(calc_id):
calc.error_message = "Failed to execute the relevant command"
else:
logger.warning(f"Calculation {calc.id} finished with return code {ret}")
+ calc.status = 3
if calc.error_message == "":
- calc.status = 3
calc.error_message = "Unknown termination"
calc.date_finished = timezone.now()
@@ -4065,10 +4880,14 @@ def get_calc(calc_id):
1, round((calc.date_finished - calc.date_started).total_seconds())
)
+ calc.save()
+
if calc.step.creates_ensemble:
analyse_opt(calc.id)
+ calc.refresh_from_db()
load_output_files(calc)
+ calc.refresh_from_db()
global cache_ind
@@ -4244,6 +5063,7 @@ def kill_calc(calc):
if calc.status not in [2, 3]:
calc.set_as_cancelled()
return
+ logger.info(f"Trying to kill calc {calc.id}")
if calc.local:
if calc.task_id != "":
@@ -4282,17 +5102,61 @@ def ping_satellite():
)
-def create_subscription(user, duration=1):
- alloc = ResourceAllocation.objects.create(
- code=get_random_string(32),
- allocation_seconds=1000 * 60,
- note=ResourceAllocation.SUBSCRIPTION,
- ) # TODO: make dynamic
- alloc.redeem(user)
- sub = Subscription.objects.create(
- subscriber=user,
- associated_allocation=alloc,
- duration=duration,
- start_date=timezone.now(),
- end_date=timezone.now() + timezone.timedelta(days=duration * 31),
+def create_account_sub(email):
+ # The email must not be used by any other user already
+ default_pass = get_random_string()
+ user = User.objects.create_user(email, password=default_pass)
+ user.is_temporary = False
+ user.is_trial = False
+ user.save()
+
+ send_mail(
+ "Your CalcUS Cloud account",
+ 'Your CalcUS Cloud account has been created. To log in, enter this email ({email}) and the temporary password "{default_pass}" on the login page (https://calcus.cloud/accounts/login/). Make sure to change the account password immediately after logging in by going on the profile page and clicking on the red "Change password" button.',
+ "bot.calcus.cloud@gmail.com",
+ recipient_list=[email],
)
+ logger.info(
+ f"Created the account with email {email} and sent instructions to that email."
+ )
+
+ return user
+
+
+def create_subscription(
+ user, stripe_sub_id, end_date, allocation, stripe_cus_id, will_renew
+):
+ time_left = max(0, user.allocated_seconds - user.billed_seconds)
+ sub_amount = max(allocation - time_left, 0)
+
+ # Make sure we don't create a duplicate subscription and allocation
+ try:
+ sub = Subscription.objects.get(subscriber=user, end_date=end_date)
+ except Subscription.DoesNotExist:
+ alloc = ResourceAllocation.objects.create(
+ code=get_random_string(32),
+ allocation_seconds=sub_amount,
+ note=ResourceAllocation.SUBSCRIPTION,
+ )
+ alloc.redeem(user)
+ sub = Subscription.objects.create(
+ subscriber=user,
+ associated_allocation=alloc,
+ stripe_sub_id=stripe_sub_id,
+ start_date=timezone.now(),
+ end_date=end_date,
+ )
+ else:
+ logger.warning(
+ f"Received duplicate subscription creation request for user {user.id} ({stripe_sub_id})"
+ )
+ return
+
+ user.refresh_from_db()
+ if user.stripe_cus_id != "" and user.stripe_cus_id != stripe_cus_id:
+ logger.warning(
+ f"Changing the Stripe customer id of user {user.id} from {user.stripe_cus_id} to {stripe_cus_id}"
+ )
+ user.stripe_cus_id = stripe_cus_id
+ user.stripe_will_renew = will_renew
+ user.save()
diff --git a/frontend/templates/best_practices/conformational_search.html b/frontend/templates/best_practices/conformational_search.html
new file mode 100755
index 00000000..250eec41
--- /dev/null
+++ b/frontend/templates/best_practices/conformational_search.html
@@ -0,0 +1,626 @@
+{% extends 'frontend/base.html' %} {% load static %} {% block extrahead %}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Conformational Sampling
+
+
+{% endblock %}
+{% block content %}
+
+
+
Conformational Sampling
+
What conformational sampling is and why it is important
+
+ In computational chemistry, conformational sampling plays a crucial role in understanding the properties and reactivity of molecules. Conformational sampling refers to the exploration of different three-dimensional arrangements, or conformations, that a molecule can adopt. This spacial arrangement can greatly influence the properties of interest because of differences in the distances between atoms, in orbital interactions and in steric constraints, among other factors.
+
+
+Molecules are exist in dynamic environments, constantly undergoing thermal motion and fluctuating between a range of conformations. What we mean by "conformation" is a local minimum in terms of all molecular coordinates, the bottom of a potential well on the potential energy surface (PES). In reality, molecules do not adopt strictly this conformation, but oscillates around this local minimum until it crosses over to another local minimum.
+
+
+In the context of computational chemistry, we often use conformations (the minima) in order to study the properties of the molecule in its state of flux. However, determining the most energetically favorable or relevant conformations for a given molecule is a complex task due to the high dimensionality of the conformational space, namely $3N - 6$ dimensions (or $3N - 5$ for linear molecules). Conformational sampling methods aim to address this challenge by efficiently exploring this vast space and identifying all the significant local minima.
+
+
+
+
+
+ Butane has only two unique stable conformations. The most stable conformation is obtained when the carbon backbone forms a zig-zag, putting the terminal methyl groups anti to each other (State A below). In this conformation, all the hydrogens are staggered to each other and the methyl groups are as far from each other as they can be.
+
+
+ The second stable conformation also has every group staggered to each other, but the methyl groups are gauche to each other (State C ). To go from one conformation to the other, butane must pass over an energy barrier (State B ). The maximum point of the barrier (the transition state) occurs when the groups of the internal carbons are eclipsed and thus repulse each other the most. An even greater barrier occurs at State D due to the eclipsed methyl groups.
+
+
+ Only one molecular coordinate is considered in this example, but larger flexible molecules have many molecular coordinates that must be explored. For example, if we consider longer linear alkane chains, the number of conformations featuring only staggered groups increases exponentially. Moreover, functional groups that can interact favorably with each other can force chains to fold due to a stability gain greater than the energy cost of bringing groups gauche to each other. The most notable example is the case of proteins, which fold into very specific structures due to internal hydrogen bonds.
+
+
+
+
+
+ Manual Animation
+
+
+
+
+
How conformations are found
+
+Various algorithms and techniques have been developed to explore the conformational space effectively. These include systematic search methods, such as the systematic torsion angle sampling or grid-based techniques, as well as stochastic search methods, like Monte Carlo methods and molecular dynamics simulations. Each approach has its strengths and limitations, and the choice of method depends on the specific objectives and computational resources available.
+
+
+ One relatively recent solution to this problem has been developed and made freely available by the group of Stefan Grimme in a software called CREST . This approach makes extensive use of the extremely fast GFNn -xTB methods (available through the xtb package ), which enable dynamics simulations for all the supported elements (hydrogen to radon). These simulations could be done using standard forcefields instead of GFNn -xTB, but this is very restrictive in terms of the supported molecules. Indeed, most forcefields have been developed with peptides and proteins in mind. Other molecules can be simulated using forcefields if proper parameters can be developed for these specific molecules. The GFNn -xTB methods thus provide a very convenient way to simulate the dynamics of almost any molecule. Moreover, the Grimme group have developed a forcefield method inspired from the GFNn -xTB methods and developed an automated parameterization procedure in order to keep the same generality. Among the three GFNn -xTB methods (n from 0 to 2), GFN2-xTB is the most accurate, and thus we will only consider this method out of the three here.
+
+
+Conformational sampling is achieved by using molecular dynamics simulations to find new conformers. Starting from the initial conformation of the molecule, multiple other conformations will be explored throughout the simulations, purely through the random atomic movements within the molecule. With long enough simulations, all conformations will be visited by the molecule.
+
+
+ However, depending on the flexibility of the molecule and the conformational change barriers, the required simulation time might be unreasonable in these conditions. In order to accelerate the exploration of new conformations, an artificial potential is periodically added around the current configuration. This potential increases the potential energy of the geometries that have been explored and thus "lowers" the barriers to new conformations. This can be seen as pouring a bucket of sand in the potential well and reducing its depth. After this has been done enough times, the potential well will be very shallow and the molecule will exit it towards new conformations. This type of dynamics is called meta-dynamics and is extensively used in the CREST package. Meta-dynamics are discussed in more detail in S. Grimme, J. Chem. Theory Comput., 2019, 155, 2847-2862.
+
+
+ The full conformational sampling procedure is explained in details on the CREST documentation . In brief, meta-dynamics simulations are used to iteratively explore more and more possible conformations until no lower energy conformation is found. The structures that are found are reoptimized and filtered down to unique conformations. Normal molecular dynamics are launched starting from the most stable conformers in order to explore nearby conformations. Finally, the new orientations of parts of the molecule are applied to other conformers in order to generate a hybrid conformation, which is likely of similar energy as its parents. This is called "Genetic Z-matrix crossing", as it uses an approach related to genetic algorithms to combined conformations into new ones without relying on dynamics simulation.
+
+
The computational cost of conformational sampling
+
+ The time required to perform the conformational sampling increases rapidly with the number of atoms in the molecule and its flexibility. To give you an idea of the resources required for your molecules, we have benchmarked the conformational search of very flexible molecules (linear alkanes) and very rigid molecules (insaturated polycycles). The GFN2-xTB calculations also used the analytical linearized Poisson-Boltzmann (ALPB) implicit solvation model to represent a typical use case (see S. Ehlert, M. Stahn, S. Spicher, S. Grimme, J. Chem. Theory Comput., 2021, 17, 4250-4261 ). The calculations were performed using 8 vCPU cores on CalcUS Cloud. The time reported is the computing time used, which is 8 times greater than the real elapsed time.
+
+
+
+
+
+ Name
+ Number of atoms
+ CPU Time (GFN-FF)
+ CPU Time (GFN2-xTB/ALPB(H2O))
+ Number of conformers*
+
+
+
+
+ Butane
+ 14
+ 80
+ 400
+ 2
+
+
+ Heptane
+ 23
+ 192
+ 2008
+ 16-17
+
+
+
+ Decane
+ 32
+ 552
+ 8040
+ 33-48
+
+
+
+ Hexadecane
+ 50
+ 10304
+ 101488
+ 143-197
+
+
+
+ Eicosane
+ 62
+ 80264
+ -
+ 999
+
+
+ Benzene
+ 12
+ 64
+ 400
+ 1
+
+
+ Biphenyl
+ 22
+ 96
+ 1136
+ 1-2
+
+
+ Pyrene
+ 26
+ 128
+ 2120
+ 1
+
+
+ Coronene
+ 36
+ 336
+ 4200
+ 1
+
+
+ Bicoronene
+ 70
+ 688
+ 19112
+ 1-2
+
+
+
+
+
+
+
+
+ While the conformational sampling of small molecules like butane and benzene required similar durations, the different quickly increases. In the extreme case, we can compare eicosane ($C_{20}H_{42}$), which took 22 CPU-hours with GFN-FF, to bicoronene ($C_{48}H_{22}$), which took under 12 CPU-minutes, despite having slightly more atoms.
+
+
+ The conformational sampling compute time grows exponentially with the number of atoms, and the flexibility of the molecules dictates how fast the time will grow. The two figures below illustrate the enormous difference between the rigid insaturated hydrocarbons in blue and the flexible alkane chains in red. The two series follow quite closely an exponential curve for molecules between 12 and 70 atoms. We can thus use the trendlines as guides to predict the minimal and maximal compute times to expect from a molecule with a given number of atoms. We can hardly predict exactly how much time the conformational sampling will take, but we can be confident that it will be between these two extremes.
+
+
+
+
+
+
+ Scaling of GFN-FF (vacuum)
+
+
+
+ Scaling of GFN2-xTB with ALPB(H2O)
+
+
+
+
+
+
+
+
+
+
+
+ Number of atoms
+ Predicted CPU Time (GFN-FF)
+ Predicted CPU Time (GFN2-xTB/ALPB(H2O))
+
+
+
+
+ 20
+ 90 — 140
+ 1000 — 1200
+
+
+ 30
+ 140 — 590
+ 2000 — 5200
+
+
+ 40
+ 210 — 2500
+ 3700 — 24,000
+
+
+ 50
+ 320 — 11,000
+ 6900 — 110,000
+
+
+ 60
+ 480 — 47,000
+ 13,000 — 500,000
+
+
+ 70
+ 730 — 200,000
+ 24,000 — 2,300,000
+
+
+
+
+
+
+
+
How unique conformations are identified
+
+ The conformational search process generates a large number of structures which may or may not correspond to known conformers. Even if two structures are visually indistinguishable to the human eye, its numerical representation (i.e. , the 3D coordinates of the atoms) will not be identical. This is because structures remain chemically identical after translations or rotations, while the numerical representation is completely different. Moreover, tiny variations in the coordinates would make the coordinates different, while humans would classify them as essentially identical.
+
+
+ In order to compare structures mathematically more rigorously, we can use their root mean squared deviation (RMSD) as difference metric. In words, this criteria expresses the average magnitude of the deviation between the position of the atoms in one structure compared to the other. It is usually implicitly assumed that both structures have been aligned (through translations and rotations) in order to obtain the smallest RMSD possible. This means that two structures whose internal coordinates are identical will have a RMSD of 0 with respect to each other. If the structures have slightly different bond distances or different angles within the molecule, the RMSD will quantify to what extent the structures differ.
+
+
+
+
+ RMSD = $\sqrt{\frac{\sum_{i=1}^N \left((x_{Ai} - x_{Bi})^2 + (y_{Ai} - y_{Bi})^2 + (z_{Ai} - z_{Bi})^2\right)}{N}}$
+
+
+
+
+
+
+
+
+ RMSD between the GFN-FF structure (red) and GFN2-xTB structure (blue)
+
+
+ Both structures were aligned in order to minimize the RMSD. Due to the different N-O bond distances and O-N-O angles, the atoms are slightly offset. In this case, the RMSD is 0.125 Å, which is quite small.
+
+
+
+
+
+
+ In the context of conformational sampling, we use a certain threshold RMSD above which two structures are considered to be different conformations. By default, the RMSD threshold of conformational searches on CalcUS Cloud is 2.0 Å, which is generally suitable for molecules with average flexibility and moderate size. However, if you wish to change this threshold, you can use the calculation specification --rthr NEWVALUE when launching the calculation. Additional available options are explained in the CalcUS documentation .
+
+
+ There are however some caveats with respect to the RMSD. It requires all the atoms to be distinguishable, meaning that we know with which atom in structure B we should compare each atom in structure A. While this is generally not a problem, it can cause duplicate conformers if there are equivalent atoms in the molecule. For example, a methyl group has three equivalent hydrogen atoms. If the methyl group rotates by 120 degrees, the structure is chemically identical, since there is still three hydrogen atoms at the same positions. However, the indices of these atoms are different, meaning that the RMSD will compare the positions of the "wrong" hydrogen atoms and will not be null. It is thus possible that these two structures will be considered to be two unique conformations, although this is chemically false.
+
+
+
General Recipe
+
+
+
+
+ Select your input structure on the launch page, either through the online sketcher or by uploading the structure in a suitable format.
+
+
+ Specify the molecule name, the solvent (if desired), the project and pick the xtb software. The calculation type should be "Conformational Search".
+
+
+ Decide on the best method to use. Refer to the table above to estimate the approximate CPU time required with each method. If your system contains transition metals or exotic valence, it would be wise to optimise the structure using GFN-FF to verify if its structure changes significantly. If it does, GFN-FF might not be suitable. If the computing time is reasonable, using GFN2-xTB is a safe choice.
+
+
+ If desired, add specifications.
+
+
+ Submit the calculation and wait for it to finish. Conformational searches do not start immediately like other xtb calculations, since they are executed on an entire virtual machine, which takes some time to start up.
+
+
+ Once the calculation is done, click on the calculation order to view the ensemble page. You will see a table with all the unique conformers ranked by energy.
+
+
+ Use the ensemble (or a subset of it) for further calculations (e.g. , frequency calculation to calculate the free energy). The Boltzmann-weighted ensemble averages will be calculated for energetic properties (energy and free energy). These values are more representative of the true value of the molecule than any individual conformation, since they take into account all (unique) significant conformations.
+
+
+
+
+
+
Tips and guidelines
+
+
+
+ The more polar the solvent, the more implicit solvation is important. Without implicit solvation, the molecule will tend to curl up in order to maximize intramolecular interactions. Implicit solvation adds the solvent-solute interactions which stabilize linear or "open" conformations. The increase in compute time caused by adding implicit solvation is pretty moderate.
+
+
+ If you used GFN-FF, it is generally recommended to reoptimize all the structures using GFN2-xTB. You can launch the optimization for all structures by clicking on "Launch calculation on ensemble". You can also only consider the relevant conformers by holding the Control key and clicking on table entries, then click "Launch calculation on selected structure(s)". GFN2-xTB optimizations take generally only a few seconds per structure, which is much less than the conformational search itself.
+
+
+
+
+{% endblock %}
diff --git a/frontend/templates/frontend/analyse.html b/frontend/templates/frontend/analyse.html
index d50da552..e35e4adc 100755
--- a/frontend/templates/frontend/analyse.html
+++ b/frontend/templates/frontend/analyse.html
@@ -1,5 +1,5 @@
{% extends 'frontend/base.html' %} {% load static %} {% block extrahead %}
-
+
+
+
{% block extrahead %} {% endblock %}
+
+ {% if IS_CLOUD %}
+ {% autoescape off %}
+ {% get_extra_body_code %}
+ {% endautoescape %}
+ {% endif %}