Skip to content

Commit fd45511

Browse files
SoniaScardSoniaScard
committed
Merge branch 'feature-config' into release-3.0
2 parents acec971 + b237df5 commit fd45511

File tree

8 files changed

+377
-0
lines changed

8 files changed

+377
-0
lines changed

‎etc/config/cert.sh‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#
2+
# Ophidia Server
3+
# Copyright (C) 2012-2023 CMCC Foundation
4+
#
5+
# This program is free software: you can redistribute it and/or modify
6+
# it under the terms of the GNU General Public License as published by
7+
# the Free Software Foundation, either version 3 of the License, or
8+
# (at your option) any later version.
9+
#
10+
# This program is distributed in the hope that it will be useful,
11+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
# GNU General Public License for more details.
14+
#
15+
# You should have received a copy of the GNU General Public License
16+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
17+
#
18+
19+
#!/bin/bash
20+
openssl req -newkey rsa:1024 \
21+
-passout pass:abcd \
22+
-subj "/" -sha1 \
23+
-keyout rootkey.pem \
24+
-out rootreq.pem
25+
openssl x509 -req -in rootreq.pem \
26+
-passin pass:abcd \
27+
-sha1 -extensions v3_ca \
28+
-signkey rootkey.pem \
29+
-out rootcert.pem
30+
cat rootcert.pem rootkey.pem > cacert.pem
31+
32+
openssl req -newkey rsa:1024 \
33+
-passout pass:abcd \
34+
-subj "/" -sha1 \
35+
-keyout serverkey.pem \
36+
-out serverreq.pem
37+
openssl x509 -req \
38+
-in serverreq.pem \
39+
-passin pass:abcd \
40+
-sha1 -extensions usr_cert \
41+
-CA cacert.pem \
42+
-CAkey cacert.pem \
43+
-CAcreateserial \
44+
-out servercert.pem
45+
cat servercert.pem serverkey.pem rootcert.pem > myserver.pem

‎etc/config/my.cnf.template‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
[client]
2+
port = $PORT
3+
socket = $PREFIX/$MYSQLENV/var/run/mysqld/mysqld.sock
4+
###user = $USER
5+
###password = $PASSWORD
6+
7+
8+
[mysqld_safe]
9+
log-error = $PREFIX/$MYSQLENV/var/log/mysql/mysql.log
10+
pid-file = $PREFIX/$MYSQLENV/var/run/mysqld/mysqld.pid
11+
12+
[mysqld]
13+
# Basic Settings
14+
bind-address = 0.0.0.0
15+
user = mysql
16+
pid-file = $PREFIX/$MYSQLENV/var/run/mysqld/mysqld.pid
17+
socket = $PREFIX/$MYSQLENV/var/run/mysqld/mysqld.sock
18+
port = $PORT
19+
basedir = $PREFIX/$MYSQLENV
20+
datadir = $PREFIX/$MYSQLENV/var/lib/mysql
21+
tmpdir = $PREFIX/$MYSQLENV/tmp
22+
plugin_dir = $PREFIX/$MYSQLENV/ophidia-primitives/lib
23+
24+
lc-messages-dir = $PREFIX/$MYSQLENV/share/mysql
25+
26+
27+
# CACHES AND LIMITS #
28+
connect_timeout = 60
29+
max_connections = 1024
30+
max_user_connections = 0
31+
max_allowed_packet = 10M
32+
open_files_limit = 4096
33+
34+
# Disabling symbolic-links is recommended to prevent assorted security risks
35+
symbolic-links = 0
36+
37+
# Recommended in standard MySQL setup
38+
sql_mode = NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES
39+
40+
# LOGGING
41+
general_log_file = $PREFIX/$MYSQLENV/var/log/mysql/mysql.log
42+
log-error = $PREFIX/$MYSQLENV/var/log/mysql/mysql-error.log

‎etc/config/mysql.sql.template‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
ALTER USER 'root'@'localhost' IDENTIFIED BY '$PASSWORD';
2+
CREATE USER 'root'@'%' IDENTIFIED BY '$PASSWORD';
3+
GRANT ALL privileges ON *.* TO 'root'@'%' WITH GRANT OPTION;
4+
GRANT SUPER on *.* to 'root'@'%';
5+
GRANT EXECUTE on mysql.* to 'root'@'%';
6+
FLUSH PRIVILEGES;
7+
8+
CREATE DATABASE ophidiadb;
9+
CREATE DATABASE oph_dimensions;
10+
CREATE USER '$USER'@'%' IDENTIFIED BY '$PASSWORD';
11+
GRANT ALL PRIVILEGES ON ophidiadb.* TO '$USER'@'%';
12+
GRANT ALL PRIVILEGES ON oph_dimensions.* TO '$USER'@'%';
13+
GRANT SUPER on *.* to '$USER'@'%';
14+
GRANT TRIGGER ON ophidiadb.* TO '$USER'@'%';
15+
FLUSH PRIVILEGES;
16+
SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));

‎etc/config/ophidia-conf.sh‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
#
2+
# Ophidia Server
3+
# Copyright (C) 2012-2023 CMCC Foundation
4+
#
5+
# This program is free software: you can redistribute it and/or modify
6+
# it under the terms of the GNU General Public License as published by
7+
# the Free Software Foundation, either version 3 of the License, or
8+
# (at your option) any later version.
9+
#
10+
# This program is distributed in the hope that it will be useful,
11+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
# GNU General Public License for more details.
14+
#
15+
# You should have received a copy of the GNU General Public License
16+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
17+
#
18+
19+
#!/bin/bash
20+
set -e
21+
myhost=`hostname`
22+
myport=3306
23+
myuser=ophidia
24+
mypassword=ophidia2022
25+
mymemory=16384
26+
numnodes=1
27+
export PREFIX=/scratch/shared/
28+
export MYSQLENV=mysql-env
29+
export CNFDIR=$(realpath "$(dirname "$0")")
30+
export TMP=$PREFIX/tmp;
31+
export SPACK_USER_CACHE_PATH=$PREFIX/spack/tmp;
32+
sed -i "s|# build_jobs: 16|build_jobs: 2|g" $SPACK_ROOT/etc/spack/defaults/config.yaml
33+
echo "[LOG] OPHIDIA COMPONENTS INSTALLATION"
34+
spack install ophidia-primitives
35+
spack install ophidia-io-server
36+
spack install ophidia-analytics-framework
37+
spack install ophidia-server
38+
export OPHIDIA_PRIMITIVES=`spack location -i ophidia-primitives`
39+
export OPHIDIA_IOSERVER=`spack location -i ophidia-io-server`
40+
export OPHIDIA_FRAMEWORK=`spack location -i ophidia-analytics-framework`
41+
export OPHIDIA_SERVER=`spack location -i ophidia-server`
42+
echo "[LOG] CONDA ENVIRONMENT CREATION"
43+
wget https://repo.anaconda.com/archive/Anaconda3-2022.10-Linux-x86_64.sh
44+
bash Anaconda3-2022.10-Linux-x86_64.sh
45+
conda create python=3.7 --prefix=$PREFIX/$MYSQLENV -y
46+
source $CONDA_PREFIX/etc/profile.d/conda.sh
47+
conda activate $PREFIX/$MYSQLENV
48+
49+
echo "[LOG] PYOPHIDIA INSTALLATION IN THE ENVIRONMENT: $MYSQLENV"
50+
conda install -c conda-forge -y pyophidia
51+
52+
echo "[LOG] MYSQL INSTALLATION IN THE ENVIRONMENT: $MYSQLENV"
53+
conda install -y mysql
54+
55+
echo "[LOG] MYSQL CONFIGURATION"
56+
cd $PREFIX/$MYSQLENV
57+
mkdir etc
58+
mkdir -p var/lib/mysql
59+
mkdir -p var/log/mysql/mysql
60+
mkdir -p var/run/mysqld
61+
mkdir -p tmp
62+
mkdir ophidia-primitives
63+
cd $PREFIX/$MYSQLENV/ophidia-primitives/
64+
ln -s $OPHIDIA_PRIMITIVES/lib/ lib
65+
cp $CNFDIR/my.cnf.template $PREFIX/$MYSQLENV/etc/my.cnf
66+
sed -i "s|\$PORT|${myport}|g" $PREFIX/$MYSQLENV/etc/my.cnf
67+
sed -i "s|\$USER|${myuser}|g" $PREFIX/$MYSQLENV/etc/my.cnf
68+
sed -i "s|\$PASSWORD|${mypassword}|g" $PREFIX/$MYSQLENV/etc/my.cnf
69+
sed -i "s|\$PREFIX|$PREFIX|g" $PREFIX/$MYSQLENV/etc/my.cnf
70+
sed -i "s|\$MYSQLENV|$MYSQLENV|g" $PREFIX/$MYSQLENV/etc/my.cnf
71+
72+
echo "[LOG] STARTING MYSQL SERVER..."
73+
cd $PREFIX/$MYSQLENV
74+
bin/mysqld --defaults-file=$PREFIX/$MYSQLENV/etc/my.cnf --user=$USER --initialize-insecure
75+
bin/mysqld_safe --defaults-file=$PREFIX/$MYSQLENV/etc/my.cnf &
76+
sleep 5
77+
cp $CNFDIR/mysql.sql.template $CNFDIR/mysql.sql
78+
sed -i "s|\$USER|${myuser}|g" $CNFDIR/mysql.sql
79+
sed -i "s|\$PASSWORD|${mypassword}|g" $CNFDIR/mysql.sql
80+
bin/mysql -u root --skip-password < $CNFDIR/mysql.sql
81+
sed -i "s|###||g" $PREFIX/$MYSQLENV/etc/my.cnf
82+
chmod 600 $PREFIX/$MYSQLENV/etc/my.cnf
83+
84+
bin/mysql --defaults-file=$PREFIX/$MYSQLENV/etc/my.cnf ophidiadb < $OPHIDIA_FRAMEWORK/etc/ophidiadb.sql
85+
spack load gsl
86+
bin/mysql -u root -p${mypassword} mysql < $OPHIDIA_PRIMITIVES/etc/create_func.sql
87+
88+
echo "[LOG] CHECK SRUN INSTALLATION"
89+
if ! command -v srun > $CNFDIR/slurm_path.txt
90+
then
91+
if ! spack location -i slurm
92+
then
93+
$CNFDIR/slurm-conf.sh
94+
fi
95+
export SLURM_PATH=`spack location -i slurm`
96+
else
97+
SRUN_PATH=$(cat $CNFDIR/slurm_path.txt)
98+
tmp=$(dirname $SRUN_PATH)
99+
export SLURM_PATH=${tmp%/*}
100+
fi
101+
echo "[LOG] SLURM INSTALLED IN $SLURM_PATH"
102+
echo "[LOG] OPHIDIA SERVER CONFIGURATION"
103+
sed -i "s|127.0.0.1|${myhost}|g" $OPHIDIA_SERVER/etc/ophidiadb.conf
104+
sed -i "s|3306|${myport}|g" $OPHIDIA_SERVER/etc/ophidiadb.conf
105+
sed -i "s|root|${myuser}|g" $OPHIDIA_SERVER/etc/ophidiadb.conf
106+
sed -i "s|abcd|${mypassword}|g" $OPHIDIA_SERVER/etc/ophidiadb.conf
107+
sed -i "s|127.0.0.1|${myhost}|g" $OPHIDIA_SERVER/etc/server.conf
108+
sed -i "s|OPH_EXTRA_LOCATION=/usr/local/ophidia/extra|OPH_EXTRA_LOCATION=${SLURM_PATH}|g" $OPHIDIA_SERVER/etc/server.conf
109+
sed -i "s|OPH_IOSERVER_LOCATION=/usr/local/ophidia/oph-cluster/oph-io-server|OPH_IOSERVER_LOCATION=${OPHIDIA_IOSERVER}|g" $OPHIDIA_SERVER/etc/server.conf
110+
echo "ENABLE_CLUSTER_DEPLOYMENT=yes" >> $OPHIDIA_SERVER/etc/server.conf
111+
echo "ENABLE_CLUSTER_INCREASE=yes" >> $OPHIDIA_SERVER/etc/server.conf
112+
sed -i "s|127.0.0.1|${myhost}|g" $OPHIDIA_SERVER/etc/script/start_ioserver.sh
113+
sed -i "s|3306|${myport}|g" $OPHIDIA_SERVER/etc/script/start_ioserver.sh
114+
sed -i "s|root|${myuser}|g" $OPHIDIA_SERVER/etc/script/start_ioserver.sh
115+
sed -i "s|abcd|${mypassword}|g" $OPHIDIA_SERVER/etc/script/start_ioserver.sh
116+
sed -i "s|65001|65000|g" $OPHIDIA_SERVER/etc/script/oph_ioserver.conf.template
117+
if [ $numnodes -eq 1 ]; then
118+
sed -i "s|--exclusive||g" $OPHIDIA_SERVER/etc/script/oph_start.sh
119+
fi
120+
121+
cd $OPHIDIA_SERVER
122+
mkdir -p log
123+
mkdir -p etc/cert
124+
cd $OPHIDIA_SERVER/etc/cert
125+
cp $CNFDIR/cert.sh $OPHIDIA_SERVER/etc/cert/cert.sh
126+
./cert.sh
127+
spack stage ophidia-server
128+
spack cd ophidia-server
129+
130+
cp -r authz $OPHIDIA_SERVER
131+
132+
echo "[LOG] OPHIDIA ANALYTICS FRAMEWORK CONFIGURATION"
133+
sed -i "s|127.0.0.1|${myhost}|g" $OPHIDIA_FRAMEWORK/etc/oph_configuration
134+
sed -i "s|3306|${myport}|g" $OPHIDIA_FRAMEWORK/etc/oph_configuration
135+
sed -i "s|root|${myuser}|g" $OPHIDIA_FRAMEWORK/etc/oph_configuration
136+
sed -i "s|abcd|${mypassword}|g" $OPHIDIA_FRAMEWORK/etc/oph_configuration
137+
sed -i "s|1024|${mymemory}|g" $OPHIDIA_FRAMEWORK/etc/oph_configuration
138+
sed -i "s|127.0.0.1|${myhost}|g" $OPHIDIA_FRAMEWORK/etc/oph_soap_configuration
139+
140+
mkdir -p $HOME/.ophidia/
141+
chmod 700 $HOME/.ophidia/
142+
echo "[mysql]" >> $HOME/.my.cnf
143+
echo "user=${myuser}" >> $HOME/.my.cnf
144+
echo "password=${mypassword}" >> $HOME/.my.cnf
145+
cd $OPHIDIA_FRAMEWORK
146+
mkdir -p html/sessions
147+
mkdir -p log
148+
149+
echo "[LOG] MYSQL SERVER SHUTDOWN"
150+
cd $PREFIX/$MYSQLENV
151+
bin/mysqladmin -u root shutdown -pophidia2022
152+
conda deactivate

‎etc/config/ophidia-run.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#
2+
# Ophidia Server
3+
# Copyright (C) 2012-2023 CMCC Foundation
4+
#
5+
# This program is free software: you can redistribute it and/or modify
6+
# it under the terms of the GNU General Public License as published by
7+
# the Free Software Foundation, either version 3 of the License, or
8+
# (at your option) any later version.
9+
#
10+
# This program is distributed in the hope that it will be useful,
11+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
# GNU General Public License for more details.
14+
#
15+
# You should have received a copy of the GNU General Public License
16+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
17+
#
18+
19+
#!/bin/bash
20+
export SLURM_CONF=$HOME/.ophidia/etc/slurm.conf
21+
cd `spack location -i munge`
22+
echo "[LOG] STARTING MUNGE..."
23+
sbin/munged -S $HOME/.ophidia/var_run_munge/munge.socket.2
24+
cd `spack location -i slurm`
25+
echo "[LOG] STARTING SLURM..."
26+
sbin/slurmd
27+
sbin/slurmctld
28+
cd $CONDA_PREFIX
29+
bin/mysqld_safe --defaults-file=/scratch/shared/mysql-env/etc/my.cnf &
30+
cd `spack location -i ophidia-server`
31+
bin/oph_server -d 2>&1 > /dev/null < /dev/null &

‎etc/config/ophidiadb.conf.template‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
OPHDB_NAME=ophidiadb
2+
OPHDB_HOST=$HOST
3+
OPHDB_PORT=$PORT
4+
OPHDB_LOGIN=$USER
5+
OPHDB_PWD=$PASSWORD

‎etc/config/slurm-conf.sh‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#
2+
# Ophidia Server
3+
# Copyright (C) 2012-2023 CMCC Foundation
4+
#
5+
# This program is free software: you can redistribute it and/or modify
6+
# it under the terms of the GNU General Public License as published by
7+
# the Free Software Foundation, either version 3 of the License, or
8+
# (at your option) any later version.
9+
#
10+
# This program is distributed in the hope that it will be useful,
11+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
# GNU General Public License for more details.
14+
#
15+
# You should have received a copy of the GNU General Public License
16+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
17+
#
18+
19+
#!/bin/bash
20+
set -e
21+
export PREFIX=/scratch/shared/
22+
export CNFDIR=$(realpath "$(dirname "$0")")
23+
echo "[LOG] MUNGE AND SLURM INSTALLATION"
24+
spack install slurm
25+
spack load munge
26+
spack load slurm
27+
export MUNGE_DIR=`spack location -i munge`
28+
export SLURM_DIR=`spack location -i slurm`
29+
cd $HOME/.ophidia/
30+
mkdir -p var/log/slurm
31+
mkdir -p var/run/slurm
32+
mkdir -p var/spool/slurm
33+
mkdir -p var/spool/slurmd
34+
mkdir -p var/log/slurmctld
35+
mkdir -p etc
36+
37+
echo "[LOG] MUNGE CONFIGURATION"
38+
cd $MUNGE_DIR
39+
sbin/mungekey
40+
mkdir -p var/run
41+
mkdir -p var/run/munge
42+
chmod 700 var/log/munge
43+
chmod 700 etc/munge
44+
chmod 755 $PREFIX
45+
ln -s $MUNGE_DIR/var/run/munge $HOME/.ophidia/var_run_munge
46+
47+
echo "[LOG] SLURM CONFIGURATION"
48+
cd $SLURM_DIR
49+
cp $CNFDIR/slurm.conf.template $HOME/.ophidia/etc/slurm.conf
50+
sed -i "s|\$HOME|$HOME|g" $HOME/.ophidia/etc/slurm.conf
51+
sed -i "s|\$USER|$USER|g" $HOME/.ophidia/etc/slurm.conf
52+
PROCS=`grep -c ^processor /proc/cpuinfo`
53+
sed -i "s|\$PROCS|$PROCS|g" $HOME/.ophidia/etc/slurm.conf
54+
export SLURM_CONF=$HOME/.ophidia/etc/slurm.conf
55+
echo "[LOG] SLURM CONFIGURED"

‎etc/config/slurm.conf.template‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
ControlMachine=localhost
2+
ControlAddr=127.0.0.1
3+
AuthInfo=socket=$HOME/.ophidia/var_run_munge/munge.socket.2
4+
AuthType=auth/munge
5+
CryptoType=crypto/munge
6+
MpiDefault=none
7+
ProctrackType=proctrack/pgid
8+
ReturnToService=1
9+
SlurmctldPidFile=$HOME/.ophidia/var/run/slurm/slurmctld.pid
10+
SlurmdPidFile=$HOME/.ophidia/var/run/slurm/slurmd.pid
11+
SlurmdSpoolDir=$HOME/.ophidia/var/spool/slurmd
12+
SlurmUser=$USER
13+
SlurmdUser=$USER
14+
StateSaveLocation=$HOME/.ophidia/var/spool/slurmd
15+
SwitchType=switch/none
16+
TaskPlugin=task/none
17+
SchedulerType=sched/backfill
18+
#SelectType=select/linear
19+
SelectType=select/cons_res
20+
SelectTypeParameters=CR_CPU
21+
#FastSchedule=1
22+
AccountingStorageType=accounting_storage/none
23+
ClusterName=cluster
24+
JobCompType=jobcomp/none
25+
JobAcctGatherType=jobacct_gather/none
26+
SlurmctldDebug=3
27+
SlurmctldLogFile=$HOME/.ophidia/var/log/slurmctld/slurmctld.log
28+
SlurmdDebug=3
29+
SlurmdLogFile=$HOME/.ophidia/var/log/slurm/slurmd.log
30+
NodeName=localhost NodeAddr=127.0.0.1 State=UNKNOWN Procs=$PROCS
31+
PartitionName=main Nodes=localhost Default=YES MaxTime=INFINITE State=UP

0 commit comments

Comments
 (0)