diff --git a/.gitignore b/.gitignore index 37a56aa..cdada3a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,37 +1,6 @@ -.DS_Store -node_modules -/dist -migrations - -# Python specific -__pycache__/ -*.py[cod] - -# Sequence Files -temp/ -temp/* -/temp/* -b000027.txt - -# local env files -.env.local -*.env.*.local -*.env - -# Log files -npm-debug.log* -yarn-debug.log* -yarn-error.log* - -# Editor directories and files -.idea -.vscode -*~ -*.suo -*.ntvs* -*.njsproj -*.sln -*.sw? -*.venv - - +*.log +.env +.pytest_cache +.ruff_cache +__pycache__ +venv diff --git a/LICENSE.md b/LICENSE.md index 1979c7b..215cab9 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -15,4 +15,4 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index f95669e..fa31f2c 100644 --- a/README.md +++ b/README.md @@ -1,201 +1,54 @@ -# Numberscope - backscope +Numberscope - backscope +======================= Copyright 2020-2022 Regents of the University of Colorado. This project is licensed under the -[MIT License](https://opensource.org/licenses/MIT). See the text of the MIT -License in LICENSE.md. - -## What is backscope? - -backscope is [Numberscope's](https://numberscope.colorado.edu) back end. It is -responsible for getting sequences and other data from the [On-Line Encyclopedia -of Integer Sequences](https://oeis.org). - -## Set up backscope - -All of these instructions assume you have already cloned the backscope -repository from `https://github.com/numberscope/backscope` and are in -the top-level directory of your clone (the directory that contains this -`README.md` file). - -### Install Python - -You need a version of Python at least equal to 3.5. (If you don't have -Python, install the latest stable version.) By installing a version of -Python greater than or equal to 3.5, you should get the package -installer for Python (`pip`) and a working `venv` module for creating a -virtual environment. - -To check to see your Python version, issue the following command: - -```shell -python --version -``` - -The output should be something like "Python 3.10.8". If you see a message about -not being able to find Python, or you don't see any output, you need to -troubleshoot your Python installation. - -Depending on how you installed Python, the executable might be named `python3`. -In that case, issue the following command: - -```shell -python3 --version -``` - -In all the remaining commands, substitute either `python` or `python3` for -`[PYEXEC]` depending on which of the above worked. - -To check to see if you have a working `venv` module, issue the following -command: - -```shell -[PYEXEC] -m venv -h -``` - -You should see help for the `venv` module. - -Note that since you will (likely) be compiling the cypari Python package, you -will (likely) need a _full_ Python3 installation, including the -"development header files." To check if these files are installed, you can -execute the following (very long) command: - -```shell -[PYEXEC] -c "from distutils import sysconfig as s; from os.path import isfile; print(isfile(s.get_config_vars()['INCLUDEPY']+'/Python.h') and 'OK')" -``` - -If this command displays anything other than `OK` (such as `False` or an error -message) then your distribution is lacking these header files. You likely -will need to install the "Python development" package for your operating -system (the details of doing so are beyond the scope of these instructions). - -### Other prerequisites - -For a successfull installation of backscope, you will (likely) need a -_full_ Pari/GP installation already present on your computer, including the -documentation files. To test if the installation is present, try -executing: - -```shell -gphelp -detex factorial -``` - -You should see a description of Pari/GP's factorial function. If not, you -may need to install Pari/GP and/or additional packages related to it, -depending on your operating system. - -### Create a virtual environment and install dependencies - -1. Create your virtual environment and activate it: - - ```bash - [PYEXEC] -m venv .venv # create a new virtual env called .venv - source .venv/bin/activate - pip install -r requirements.txt - pip install --force cypari2 - ``` - - All remaining instructions assume that you have this virtual environment - activated. So if, for example, you stop and log out and come back later - and pick up the process, make sure to re-activate the virtual environment - by re-issuing the `source .venv/bin/activate` command in the top-level - directory of your backscope clone. Note also that once the virtual - environment is activated, the `python` command will invoke the proper - version of `python`, so you no longer need to worry about whether you - need to call `python3` or `python`. Hence, the remaining instructions - all just use `python`. - -2. Install and configure PostgreSQL and create an empty database: - - Specific instructions for PostgreSQL installation are unfortunately beyond - the current scope of this README, as they depend greatly on the particulars - of your operating system. You need to end up with a running - Postgres server on your machine that will accept localhost connections. - - Specifically, once you are set up, it should be possible to use the command - `psql` to connect to a Postgres shell where you can create a database for - backscope. You should be able to use the `-U` flag to specify a user who - has the correct permissions to access the Postgres shell and create a database. - - ```bash - psql -U - =# CREATE DATABASE ; - CREATE DATABASE - =# \q - ``` - -3. Set up your environment and initialize the database: - - This project uses python-dotenv. In order to detect your database - username / password, you must create a file called `.env` in the root - of your directory containing: - ``` - export APP_ENVIRONMENT="development" - export DATABASE_URI="postgresql://localhost/" - export SECRET_KEY="Uneccessary for development" - export POSTGRES_USER="" - export POSTGRES_DB="" - export POSTGRES_PASSWORD="" - ``` - You can see other configuration options inside - [the config file](./flaskr/config.py). - -4. Configure the database: - - ```bash - python manage.py db init # initializes tables inside database - ``` - - The previous command will issue a message about editing - `alembic.ini`, which is safe to ignore. The default works fine. - - ```bash - python manage.py db migrate # migrate data models - python manage.py db upgrade # upgrade changes to database - psql -U -d - db=# \d - Schema | Name | Type | Owner - --------+-----------------+----------+-------------------- - public | alembic_version | table | - public | sequences | table | - public | user | table | - public | user_id_seq | sequence | - db=# \q - ``` - -## Run backscope - -Option 1: -```bash -python manage.py runserver -``` - -Option 2: -```bash -export FLASK_APP=flaskr -flask run -``` - -This should print a series of messages. One of these -messages should be the URL the server is running on, typically -`http://127.0.0.1:5000/`. To test that the server is working correctly, -try visiting `/api/get_oeis_values/A000030/50` (substitute in the server -URL for ``). This should display the first digits of the numbers from -0 through 49. - -## More information - -### General - -- [API Endpoints](./doc/api_endpoints.md) -- [Directory Descriptions](./doc/directory_descriptions.md) -- [Resetting the Database](./doc/resetting-the-database.md) - -### Administering the production backscope instance - -- [Update Backscope](./doc/update-backscope.md) -- [Server Administration](./doc/server-administration.md) -- [Database Administration](./doc/database-administration.md) -- [server Directory README](./server/README.md) - +[MIT License](https://opensource.org/licenses/MIT). See the text of the +MIT License in LICENSE.md. + +What is backscope? +------------------ + +backscope is [Numberscope's](https://numberscope.colorado.edu) back end. +It is responsible for getting sequences and other data from the +[On-Line Encyclopedia of Integer Sequences](https://oeis.org). + +Quick start +----------- + +1. Clone this repo. +2. Install prerequisites: Git, Python 3, Python 3 dev package, + Python 3 package for creating virtual environments, full installation + of pari-gp (including all metadata files — you might need to install + a package like "libpari-dev"), GNU multi-precision arithmetic dev + package, a C compiler, and C build tools. + + For detailed instructions on installing backscope on Ubuntu, see + [this doc](doc/install-ubuntu.md). +3. Create a virtual environment: `python -m venv venv`. +4. Activate the virtual environment: `source venv/bin/activate`. +5. Install deps: `pip install -r requirements.txt`. +6. Install and configure PostgreSQL and create an empty database. + + For detailed instructions on installing and configuring PostgreSQL, + see [ths doc](doc/install-postgres.md). +7. Run backscope: `flask run` + + For detailed instructions on running backscope, see + [this doc](doc/running-backscope.md). + +More info +--------- + +If you plan on developing backscope, you might find one of these useful: + +- [How to format, lint, and test your code](doc/format-lint-test.md) +- [How to install a pre-commit hook](doc/pre-commit-hook.md) +- [How to do database migrations](doc/db-migrations.md) +- [How to reset your database](doc/db-reset.md) +- [Understanding our requirements files](doc/requirements.md) + +If you are a maintainer, you might find one of these useful: + +- [How to update frontscope on the CU server](doc/server-update-frontscope.md) +- [How to update backscope on the CU server](doc/server-update-backscope.md) +- [How to administer our server](doc/server-admin.md) +- [How to administer our database](doc/db-admin.md) \ No newline at end of file diff --git a/api.log b/api.log index e69de29..86d23b4 100644 --- a/api.log +++ b/api.log @@ -0,0 +1,52 @@ +attempting to connect to db +connected to db +attempting to connect to db +connected to db +attempting to connect to db +connected to db +attempting to connect to db +connected to db +MY_FOO = None +HTTPSConnectionPool(host='oeis.orga000040', port=443): Max retries exceeded with url: /b000040.txt (Caused by NameResolutionError(": Failed to resolve 'oeis.orga000040' ([Errno -2] Name or service not known)")) +HTTPSConnectionPool(host='oeis.orga000041', port=443): Max retries exceeded with url: /b000041.txt (Caused by NameResolutionError(": Failed to resolve 'oeis.orga000041' ([Errno -2] Name or service not known)")) +HTTPSConnectionPool(host='oeis.orga000041', port=443): Max retries exceeded with url: /b000041.txt (Caused by NameResolutionError(": Failed to resolve 'oeis.orga000041' ([Errno -2] Name or service not known)")) +attempting to connect to db +connected to db +attempting to connect to db +connected to db +MY_FOO = foo +HTTPSConnectionPool(host='oeis.orga000040', port=443): Max retries exceeded with url: /b000040.txt (Caused by NameResolutionError(": Failed to resolve 'oeis.orga000040' ([Errno -2] Name or service not known)")) +attempting to connect to db +connected to db +attempting to connect to db +connected to db +attempting to connect to db +connected to db +attempting to connect to db +connected to db +'NoneType' object has no attribute 'append' +attempting to connect to db +connected to db +attempting to connect to db +connected to db +metadata fetching for A000001 in progress +metadata fetching for A000001 in progress +metadata fetching for A000001 in progress +metadata fetching for A000001 in progress +attempting to connect to db +connected to db +attempting to connect to db +connected to db +metadata fetching for A000001 in progress +attempting to connect to db +connected to db +attempting to connect to db +connected to db +attempting to connect to db +connected to db +attempting to connect to db +connected to db +attempting to connect to db +connected to db +attempting to connect to db +connected to db diff --git a/app.py b/app.py new file mode 100644 index 0000000..19d46b0 --- /dev/null +++ b/app.py @@ -0,0 +1,142 @@ +""" +This is the entry point for backscope. It creates the Flask instance and +defines the functionality of the API. +""" + +import logging +import os +import sys +from logging.handlers import RotatingFileHandler +from dotenv import load_dotenv +from flask import Flask, jsonify, abort +from flask_migrate import Migrate +from sequence import Sequence, fetch_values, fetch_metadata, fetch_factors +from sequence import db + + +def create_app(): + app = Flask(__name__) + + # Load environment variables from the .env file into os.environ. + load_dotenv() + + # Set up file logging. + file_handler = RotatingFileHandler( + os.environ.get("LOG_FILE_NAME"), maxBytes=10000, backupCount=1 + ) + file_handler.setLevel(logging.INFO) + app.logger.addHandler(file_handler) + + # Set up stdout logging. + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setLevel(logging.DEBUG) + app.logger.addHandler(stdout_handler) + + # Connect to the database. + app.logger.info("attempting to connect to db") + app.config["SQLALCHEMY_DATABASE_URI"] = os.environ.get("DB_URI") + db.init_app(app) + app.logger.info("connected to db") + + # Establish the migrate command for database migrations via CLI. + migrate = Migrate() + migrate.init_app(app, db) + + @app.route("/") + def hello_world(): + """Dummy route. Should be removed once done developing. + + :return: string of HTML + """ + app.logger.info(f"MY_FOO = {os.environ.get('MY_FOO')}") + return "

Hello, World!

" + + @app.route("/api/get_oeis_sequence/") + def get_oeis_sequence(oeis_id): + """A route for getting a whole, raw sequence object from the DB. + + This route is useful if you want to know what data is filled in + for a given sequence object in the database. Hopefully it will + be useful in development and debugging. + + :param oeis_id: OEIS sequence ID, typically Annnnnn where n is a number + :return: JSON sequence dict, possibly empty + """ + seq = Sequence.get(oeis_id) + if seq: + return jsonify(Sequence.as_dict(seq)) + else: + return jsonify({}) + + # Make providing the number of elements you want optional. + # See https://stackoverflow.com/a/14032302/15027348. + @app.route("/api/get_oeis_values/", defaults={"num_elements": None}) + @app.route("/api/get_oeis_values//") + def get_oeis_values(oeis_id, num_elements): + """A route that gets a sequence's values from the DB or the OEIS. + + :param oeis_id: OEIS sequence ID, typically Annnnnn where n is a number + :param num_elements: (optional) number of elements the caller wants + :return: JSON object containing sequence's indices and values + """ + try: + vals = fetch_values(oeis_id) + if num_elements and num_elements < len(vals.keys()): + # Create a list of keys that the caller wants. + # Taken from https://stackoverflow.com/a/16819250/15027348. + wanted_keys = list(vals.keys())[0:num_elements] + + # Return a subset of the vals dict. + # Taken from https://stackoverflow.com/a/5352630/15027348. + return jsonify({k: vals[k] for k in wanted_keys}) + else: + return jsonify(vals) + + except Exception as e: + app.logger.error(e) + abort(500) + + @app.route("/api/get_oeis_metadata/") + def get_oeis_metadata(oeis_id): + """A route that gets a sequence's metadata from the DB or the OEIS. + + The response from this route might take a long time. Possibly + hours, depending on the popularity of the sequence. + + :param oeis_id: OEIS sequence ID, typically Annnnnn where n is a number + :return: dict containing metadata fields + """ + try: + seq = fetch_metadata(oeis_id) + return jsonify( + {"name": seq.name, "xrefs": seq.raw_refs, "backrefs": seq.backrefs} + ) + except Exception as e: + app.logger.error(e) + abort(500) + + # Make providing the number of elements you want optional. + # See https://stackoverflow.com/a/14032302/15027348. + @app.route( + "/api/get_oeis_factors/", defaults={"num_elements": None} + ) + @app.route("/api/get_oeis_factors//") + def get_oeis_factors(oeis_id, num_elements): + """A route for getting a sequence's factors. + + :param oeis_id: OEIS sequence ID, typically Annnnnn where n is a number + :param num_elements: (optional) number of elements the caller wants + :return: a list containing list of factors for each element in the sequence + """ + try: + factors = fetch_factors(oeis_id) + if num_elements and num_elements < len(factors): + return jsonify(factors[:num_elements]) + else: + return jsonify(factors) + + except Exception as e: + app.logger.error(e) + abort(500) + + return app diff --git a/doc/api_endpoints.md b/doc/api_endpoints.md deleted file mode 100644 index 2798d7c..0000000 --- a/doc/api_endpoints.md +++ /dev/null @@ -1,115 +0,0 @@ -# API endpoints - -As of 2022-11-01, this document might be outdated. - -This documents all of the endpoints provided by the backscope server. -All of them return JSON data with the specified keys and values. Also, every -endpoint includes the key 'id' with value the OEIS id for the sake of verifying -that it is the data as requested. In case of an OEIS_ID that does not match -anything in the OEIS, an error string is returned. Note that the angle brackets -<> in the URLS indicate where subsitutions are made; they should not be present -in the URLs actually used. - -Also note that if any of the requests are made for a given sequence, then the -back end will in the background obtain all of the data necessary to respond -to all of the endpoints for future requests concerning that sequence without -going back to the OEIS. Note that this background work may take an appreciable -amount of time, especially if the sequence has lots of references within the -OEIS. - -### URL: `api/get_oeis_values//` - -This is the most rapid endpoint, it makes at most one request to the OEIS server -(and only if the OEIS_ID has not previously been requested). If you are running -the server to test it on your local host, a full URL would be -`http://127.0.0.1:5000/api/get_oeis_values/A000030/50` which will return the -first digits of the numbers 0 through 49. - -#### Key: name - -A string giving the official name of the OEIS sequence with id OEIS_ID, -if already known to backscope, or a temporary name if not. - -#### Key: values - -An array of _strings_ (of digits) giving the first COUNT values of the sequence -with id OEIS_ID. Since some sequence values correspond to extremely large -numbers, strings are used to avoid the limitations of any particular numeric -datatype. - -### URL: `api/get_oeis_name_and_values/` - -This one is potentially a bit slower than the above URL, as it may make -an extra request to ensure that the name is correct. If you are running -the server on your local host, a full URL would be -`http://127.0.0.1:5000/api/get_oeis_name_and_values/A003173`, which will -return the nine Heegner numbers and their full name as an OEIS -sequence (basically, the name describes what a Heegner number is). - -#### Key: name - -A string giving the official name of the OEIS sequence with id OEIS_ID. - -#### Key: values - -An array of strings (of digits) giving all values of the sequence with id -OEIS_ID known to the OEIS. - -### URL: `api/get_oeis_metadata/` - -A potentially very slow endpoint (if the sequence is unknown to the backscope); -may make hundreds of requests to the OEIS to generate all of the back -references to the sequence. If you are running the server on your local -machine, a full URL would be -`http://127.0.0.1:5000/api/get_oeis_metadata/A028444` which will show the full -name of the Busy Beaver sequence, the one text line of sequences it -references, and the IDs of the ten sequences that refer to it. - -#### Key: name - -A string giving the official name of the OEIS sequence with id OEIS_ID. - -#### Key: xrefs - -A string which is the concatenation (separated by newlines) of all of the -OEIS text "xref" records for the sequence with id OEIS_ID. - -#### Key: backrefs - -An array of strings giving all OEIS ids that mention the given OEIS_ID. - -### URL: `api/get_oeis_factors//` - -This could take a long time. It internally does everything that the endpoint -`get_oeis_metadata` does, and then once the result is stored in the database -it proceeds to factor the first `` terms of the sequence (or all of them -if there are not that many). If you are running the server to test it on your -local host, a full URL would be -`http://127.0.0.1:5000/api/get_oeis_factors/A006862/50` which will return the -factorizations of 1 + the product of the first n primes, for n < 50. The -first 42 terms will be factored and larger terms will return `no_fac` (since -they are deemed too large to factor in a reasonable time). - -The factorization is performed by pari: `https://pari.math.u-bordeaux.fr/`. -Previous factorization requests are cached in the database for efficiency. -Returns the following data: - -#### Key: name - -A string giving the official name of the OEIS sequence with id OEIS_ID. - -#### Key: factors - -An array whose indices match the indices of the values of the sequence. -The format of each entry is a string of the form - -`[[p,e],[q,f],...]` - -where each entry `[p,e]` represents a factor of the prime p to the power e. If -an integer is negative, `[-1,1]` is included. If the integer is 1, the -factorization is `[]` (empty). If the integer is 0, the factorization -is `[[0,1]]`. Any successful factorization has the property that if you -multiply 1 times the product of `p^e` for all `[p,e]` in the array, you -obtain the original value. This format is essentially that supported by pari. -If the integer exceeds 2^200, the factorization is not attempted and -the factorization is stored as `no_fac`. \ No newline at end of file diff --git a/doc/database-administration.md b/doc/db-admin.md similarity index 85% rename from doc/database-administration.md rename to doc/db-admin.md index e9d88d2..a125009 100644 --- a/doc/database-administration.md +++ b/doc/db-admin.md @@ -1,6 +1,8 @@ -# Database administration +Database administration +======================= -## How the PostgreSQL database is set up +How the PostgreSQL database is set up +------------------------------------- When you install Postgres, there's typically a default `postgres` user and a default `postgres` database. We have maintained this setup. We @@ -15,7 +17,8 @@ you define in your code and configures a relational database to store those objects. Managing the tables, columns, etc. in the `scope` database should be done (as much as possible) by SQLAlchemy. -## PostgreSQL commands +PostgreSQL commands +------------------- Figure out who you are: ```sh @@ -23,12 +26,12 @@ you@numberscope:~$ whoami ``` Change to the `scope` user: -``` +```sh you@numberscope:~$ sudo -i -u scope ``` Enter the Postgres shell: -``` +```sh scope@numberscope:~$ psql ``` @@ -55,4 +58,4 @@ scope=# \h Exit the Postgres shell: ``` scope=# \q -``` +``` \ No newline at end of file diff --git a/doc/db-migrations.md b/doc/db-migrations.md new file mode 100644 index 0000000..035fb0d --- /dev/null +++ b/doc/db-migrations.md @@ -0,0 +1,64 @@ +Database Migrations +=================== + +We use an object relational mapper called SQLAlchemy (the package is +Flask SQLAlchemy) to describe the data we want to store in the database, +and it does the heavy lifting of interacting with the database. +Normally, you'd have to write SQL queries to insert and extract data +from the database, but ORMs abstract that work away from the developer. + +When you change the data you want to store in your database (e.g. you +change the name of a variable), you have to "migrate" (change) the +database to conform to your new data. + +Flask-Migrate is an extension that handles SQLAlchemy database +migrations for Flask applications using Alembic. The database operations +are made available through the Flask command-line interface. + +How to migrate and update your database +--------------------------------------- + +### Run the `migrate` script + +> The migration script needs to be reviewed and edited, as Alembic is +> not always able to detect every change you make to your models. In +> particular, Alembic is currently unable to detect table name changes, +> column name changes, or anonymously named constraints. A detailed +> summary of limitations can be found in the Alembic autogenerate +> documentation. Once finalized, the migration script also needs to be +> added to version control. + +``` +flask db migrate -m "Your migration message goes here." +``` + +### Run the `update` script + +Apply the changes described by the migration script to your database: + +``` +flask db upgrade +``` + +First-time setup +---------------- + +If you don't have a migrations directory (you should because it's +tracked in version control), you can create it by entering the following +command: + +``` +flask db init +``` + +More help +--------- + +To see the commands you can run: + +``` +flask db --help +``` + +For more documentation on Flask-Migrate, see +https://flask-migrate.readthedocs.io/en/latest/. \ No newline at end of file diff --git a/doc/resetting-the-database.md b/doc/db-reset.md similarity index 66% rename from doc/resetting-the-database.md rename to doc/db-reset.md index cbb34e1..9c9d79d 100644 --- a/doc/resetting-the-database.md +++ b/doc/db-reset.md @@ -1,17 +1,18 @@ -# Resetting the database +Resetting the database +====================== -If you need to clear out the entire database and start from scratch -- for -example, when pulling a commit that modifies the database schema -- the easiest +If you need to clear out the entire database and start from scratch — for +example, when pulling a commit that modifies the database schema — the easiest thing to do is delete the database and reinitialize an empty database. An example session for this is below; make sure before executing these commands that you have activated the virtual environment -(`source .venv/bin/activate`) for the backscope project. +(`source venv/bin/activate`) for the backscope project. -```bash +``` dropdb createdb rm -rf migrations -python manage.py db init -python manage.py db migrate -python manage.py db upgrade +flask db init +flask db migrate +flask db upgrade ``` \ No newline at end of file diff --git a/doc/directory_descriptions.md b/doc/directory_descriptions.md deleted file mode 100644 index 14c0466..0000000 --- a/doc/directory_descriptions.md +++ /dev/null @@ -1,79 +0,0 @@ -# Description of directories - -As of 2022-11-01, this document might be outdated. - -##### migrations - -Auto generated - -##### manage.py - -The main entry point into the application - -The main usage from this file is: - -```bash -$ python3 manage.py db init -$ python3 manage.py db migrate -$ python3 manage.py db upgrade - -$ python3 manage.py runserver -``` - -Any other target may be considered for future development - -##### api.log - -The log file for the flask application - -an example logging statment is shown below in python3 - -```python3 - app.logger.info('%s logged in successfully', user.username) -``` - -Logging levels are set in flaskr/config.py - -Refer to https://flask.palletsprojects.com/en/1.1.x/logging/ for more information - -##### flaskr - -Flaskr is the main application. It contains all entry points to the application. \_\_init\_\_.py contains all the application macros. - -##### flaskr/\_\_init\_\_.py - -The primary file for all application macros - -##### flaskr/config.py - -The configuration directory for all macro config options. Generally development mode is being used. - -##### flaskr/nscope - -The nscope api python module. This contains numberscope endpoints and blueprints and models - -##### flaskr/nscope/\_\_init\_\_.py - -The main entry point to nscope, containing the blueprint for numberscope (imported in \_\_init\_\_.py in flaskr) - -##### flaskr/nscope/models.py - -Database models. See the example for how to define models as well as https://flask-sqlalchemy.palletsprojects.com/en/2.x/models/ - -##### flaskr/nscope/views.py - -The primary blueprint for numberscope. This is the main application. Note that currently, there is only one endpoint (vuetest). But refer to the documentation within views on how to create a new route. - -### WSGI Setup - -WSGI is setup on the production server. all wsgi instances and configurations are neglected in the github repository for security. - -Documentation is in progress. - -On the server, you may manage the status of the application by the systemd entry point for numberscope - -```bash -$ sudo systemctl status numberscopeFlask.service -``` - -Numberscope is serving both index.html as well as the numberscope api routes declared inside flask. diff --git a/doc/format-lint-test.md b/doc/format-lint-test.md new file mode 100644 index 0000000..701120e --- /dev/null +++ b/doc/format-lint-test.md @@ -0,0 +1,47 @@ +Formatting, Linting, and Testing +================================ + +### Formatting + +To auto-format your code, enter the following command from the root +of the backscope directory: + +``` +black . +``` + +(The virtual environment directory used to be named `.venv`, but we +renamed it to `venv` so that files inside that directory would be +ignored by the formatter or the linter, I (Liam) can't remember which.) + +### Linting + +To lint your code, enter the following command from the root +of the backscope directory: + +``` +ruff check . +``` + +### Testing + +#### Seed the database before you run your tests + +Before you run the tests suite for the first time, you should seed +your database with data for the A000001 sequence. To do this, hit the +API routes for the sequence object, values, and metadata in that order. +It is important to do this because at least one test (as of this +writing, only the `test_get_oeis_sequence` test) assumes there is data +in the database. (Otherwise it wouldn't be able to test much.) + +If you don't seed your database with data for the A000001 sequence, +right now it is possible to make all the tests pass by simply running +the tests suite again. However, this might change if future tests assume +the presence of data before the database is seeded. + +To run the tests suite, enter the following command from the root +of the backscope directory: + +``` +pytest +``` \ No newline at end of file diff --git a/doc/install-postgres.md b/doc/install-postgres.md new file mode 100644 index 0000000..c736933 --- /dev/null +++ b/doc/install-postgres.md @@ -0,0 +1,61 @@ +Install and configure PostgreSQL +================================ + +Comprehensive instructions for PostgreSQL installation are unfortunately +beyond the current scope of this README, as they depend greatly on the +particulars of your operating system. You need to end up with a running +Postgres server on your machine that will accept localhost connections. + +Specifically, once you are set up, it should be possible to use the +command `psql` to connect to a Postgres shell where you can create a +database for backscope. You should be able to use the `-U` flag to +specify a user who has the correct permissions to access the Postgres +shell and create a database. + +``` +psql -U +=# CREATE DATABASE ; +CREATE DATABASE +=# \q +``` + +### Set up your environment and initialize the database + +This project uses python-dotenv. In order to detect your database +username / password, you must create a file called `.env` in the root +of your directory containing: + +``` +DB_PASSWORD="" +DB_URI=postgresql://localhost/backscope +DB_USER="" +LOG_FILE_NAME=api.log +OEIS_URL=https://oeis.org/ +``` + +### Configure the database + +``` +flask db init # initializes tables inside database +``` + +The previous command might issue a message about editing +`alembic.ini`, which is safe to ignore. The default works fine. + +``` +flask db migrate # migrate data models +flask db upgrade # upgrade changes to database +psql -U -d +db=# \d + List of relations + Schema | Name | Type | Owner +--------+-----------------+-------+------- + public | alembic_version | table | + public | sequences | table | +db=# \q +``` + +### Database migrations + +For further instructions on database migrations, see +[this doc](db-migrations.md). \ No newline at end of file diff --git a/doc/install-ubuntu.md b/doc/install-ubuntu.md new file mode 100644 index 0000000..f5aeffa --- /dev/null +++ b/doc/install-ubuntu.md @@ -0,0 +1,189 @@ +Installing backscope on Ubuntu +============================== + +These instructions are for Ubuntu, a Linux distribution. If you are +trying to run backscope on a different Linux distribution or on a +different operating system, you will need to modify the commands and/or +the steps. + +### Install Git + +First, check if Git is installed: + +``` +which git +``` + +If you don't see any output: + +``` +sudo apt install git +``` + +### Clone backscope + +If you are using HTTP: + +``` +git clone https://github.com/numberscope/backscope.git +``` + +If you are using SSH: + +``` +git clone git@github.com:numberscope/backscope.git +``` + +### Install pari-gp, required for cypari2 + +This is the actual PARI/GP package. + +``` +sudo apt install pari-gp +``` + +To verify that it was installed correctly, try to check the version of +`gp`: + +``` +gp --version +``` + +If you see some output about the version number, then you have likely +installed the package correctly. + +### Install libpari-dev, which contains a file needed to install cypari2 + +This is the PARI library development package. It contains a `pari.desc` +file which is crucial for cypari2. + +``` +sudo apt install libpari-dev +``` + +Ensure the file exists: + +``` +ls -al /usr/share/pari +``` + +You should see a `pari.desc` file in that directory. + +### Install libgmp-dev, required for cypari2 + +This is the package for the GNU multi-precision arithmetic library +developer tools. + +``` +sudo apt install libgmp-dev +``` + +### Install essential build tools, required for cypari2. + +Essential build tools are used when we compile cypari2. The tools that +are installed are gcc, g++, gdb, etc. — the generic C/C++ toolkit. + +``` +sudo apt install build-essential +``` + +### Install Python 3 + +You need a version of Python at least equal to 3.5. (If you don't have +Python, install the latest stable version.) By installing a version of +Python greater than or equal to 3.5, you should get the package +installer for Python (`pip`) and a working `venv` module for creating a +virtual environment. + +To check to see your Python version, issue the following command: + +```shell +python --version +``` + +The output should be something like "Python 3.10.8". If you see a message about +not being able to find Python, or you don't see any output, you need to +troubleshoot your Python installation. + +Depending on how you installed Python, the executable might be named `python3`. +In that case, issue the following command: + +```shell +python3 --version +``` + +In all the remaining commands, substitute either `python` or `python3` for +`[PYEXEC]` depending on which of the above worked. + +To check to see if you have a working `venv` module, issue the following +command: + +```shell +[PYEXEC] -m venv -h +``` + +You should see help for the `venv` module. + +Note that since you will (likely) be compiling the cypari2 Python +package, you will (likely) need a _full_ Python 3 installation, including +the "development header files." To check if these files are installed, +you can execute the following (very long) command: + +```shell +[PYEXEC] -c "from distutils import sysconfig as s; from os.path import isfile; print(isfile(s.get_config_vars()['INCLUDEPY']+'/Python.h') and 'OK')" +``` + +If this command displays anything other than `OK` (such as `False` or an error +message) then your distribution is lacking these header files. + +### Install python3-dev, required for cypari2 + +This is the Python development package. We need it to compile cypari2. + +``` +sudo apt install python3-dev +``` + +### Install the package that makes it so you can create a virtual environment + +``` +sudo apt install python3.xy-venv +``` + +`xy` is a version number, e.g. `python3.10-venv`. + +### Create the virtual environment. + +``` +python3 -m venv venv +``` + +### Activate the virtual environment + +If you are using Bash: + +``` +source .venv/bin/activate +``` + +(If you are using a shell other than Bash, there might be an activate +script in the `.venv/bin/` directory for your shell.) + +All remaining instructions assume that you have this virtual environment +activated. So if, for example, you stop and log out and come back later +and pick up the process, make sure to re-activate the virtual environment +by re-issuing the `source .venv/bin/activate` command in the top-level +directory of your backscope clone. Note also that once the virtual +environment is activated, the `python` command will invoke the proper +version of `python`, so you no longer need to worry about whether you +need to call `python3` or `python`. Hence, the remaining instructions +all just use `python`. + +### Install dependencies + +This installs all of backscope's dependencies listed in +`requirements.txt`. + +``` +pip install -r requirements.txt +``` \ No newline at end of file diff --git a/doc/pre-commit-hook.md b/doc/pre-commit-hook.md new file mode 100644 index 0000000..135e32f --- /dev/null +++ b/doc/pre-commit-hook.md @@ -0,0 +1,10 @@ +# `pre-commit` Hook + +The `pre-commit` Git hook is a Bash script that runs before you make +a Git commit. It lives in the `tools` directory. You should install +it before you start developing `backscope`. To do so: + +``` +chmod +x tools/pre-commit +cp tools/pre-commit .git/hooks/ +``` \ No newline at end of file diff --git a/doc/requirements.md b/doc/requirements.md new file mode 100644 index 0000000..182a51f --- /dev/null +++ b/doc/requirements.md @@ -0,0 +1,20 @@ +Requirements +============ + +The `requirements.txt` file should only contain "top-level" requirements +for backscope. That is, if we explicitly want a requirement, we list it +in `requirements.txt`. If a requirement has dependencies, those are +listed in `requirements-freeze.txt`. To generate +`requirements-freeze.txt`, you can enter the following command if you +have the virtual environment activated in a Posix shell: + +``` +pip freeze > requirements-freeze.txt +``` + +Ideally, this file should be generated every time a "top-level" +dependency is installed. + +The `requirements.txt` file is analogous to frontscope's `package.json`, +and the `requirements-freeze.txt` is analogous to frontscope's +`package-lock.json`. diff --git a/doc/running-backscope.md b/doc/running-backscope.md new file mode 100644 index 0000000..e76814c --- /dev/null +++ b/doc/running-backscope.md @@ -0,0 +1,17 @@ +Running backscope +================= + +To run the development server: + +``` +flask run +``` + +By enabling debug mode, the server will automatically reload if code +changes, and will show an interactive debugger in the browser if an +error occurs during a request. To run the development server in "debug" +mode: + +``` +flask run --debug +``` diff --git a/doc/server-administration.md b/doc/server-admin.md similarity index 94% rename from doc/server-administration.md rename to doc/server-admin.md index 40f7daa..21637f4 100644 --- a/doc/server-administration.md +++ b/doc/server-admin.md @@ -1,10 +1,12 @@ -# Server administration +Server administration +===================== As of this writing, `backscope` is running on a server in CU's math building. For info on the point of contact for the server, ask one of Numberscope's maintainers. -## How `backscope` is set up +How `backscope` is set up +------------------------- In the `/home` directory, there's a directory `scope` for the user `scope`. Within the `scope` directory, there's a `repos` directory. @@ -29,7 +31,8 @@ symlinked `numberscope.service` file that runs the `production.sh` script. (The `/etc/systemd/system/` is a directory that houses systemd (system daemon) files.) -## `numberscope` systemd commands +`numberscope` systemd commands +------------------------------ Note: We named the systemd file `numberscope.service` because it is responsible for serving `frontscope`'s built files as well as forwarding @@ -55,7 +58,8 @@ Stop `numberscope`: sudo systemctl stop numberscope ``` -## How Nginx is set up +How Nginx is set up +------------------- As of this writing, our Nginx configuration is simple. We preserve the default configuration we get from Nginx upon installation with two @@ -69,7 +73,8 @@ exceptions: 2. We remove the `default` site from the `/etc/nginx/sites-enabled` directory. -## Note on HTTPS +Note on HTTPS +------------- We used Certbot to create SSL certificates so that we can access numberscope.colorado.edu using HTTPS. In doing so, some of the Nginx @@ -77,7 +82,8 @@ configuration files were modified. Certbot seems to insert a comment when it modifies your configuration files, so it should be obvious what the Certbot modifications are. -## Steps for using Certbot +Steps for using Certbot +----------------------- These are the steps followed to create SSL certificates. They can be found @@ -165,4 +171,4 @@ locations: To confirm that your site is set up properly, visit https://yourwebsite.com/ in your browser and look for the lock icon in -the URL bar. +the URL bar. \ No newline at end of file diff --git a/doc/update-backscope.md b/doc/server-update-backscope.md similarity index 88% rename from doc/update-backscope.md rename to doc/server-update-backscope.md index f5febd4..b45dc10 100644 --- a/doc/update-backscope.md +++ b/doc/server-update-backscope.md @@ -1,4 +1,5 @@ -# How to update backscope on the colorado.edu server +How to update backscope on the colorado.edu server +================================================== This assumes the latest changes have been thoroughly tested. When you restart `backscope`, a script activates the virtual environment and diff --git a/doc/update-frontscope.md b/doc/server-update-frontscope.md similarity index 89% rename from doc/update-frontscope.md rename to doc/server-update-frontscope.md index e60a287..b36a234 100644 --- a/doc/update-frontscope.md +++ b/doc/server-update-frontscope.md @@ -1,4 +1,5 @@ -# How to update frontscope on the colorado.edu server +How to update frontscope on the colorado.edu server +=================================================== 1. Connect to the server: `ssh you@numberscope.colorado.edu` 2. Change user: `sudo -i -u scope` diff --git a/flaskr.ini b/flaskr.ini deleted file mode 100644 index db490f7..0000000 --- a/flaskr.ini +++ /dev/null @@ -1,11 +0,0 @@ -[uwsgi] -module = wsgi:app - -master = true -processes = 4 - -socket = project.sock -chmod-socket = 777 -vacuum = true - -die-on-term = true diff --git a/flaskr/__init__.py b/flaskr/__init__.py deleted file mode 100644 index dbf7e55..0000000 --- a/flaskr/__init__.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -Init file (creates app and database) -""" - -import os -from flask import Flask -import click -from flask.cli import with_appcontext -from flask_sqlalchemy import SQLAlchemy -from flask_cors import CORS -import logging -from logging.handlers import RotatingFileHandler -import sys - -from dotenv import load_dotenv - -from .config import config - -# This statement loads all environment variables from .env -load_dotenv() - -# Create a new sql alchemy database object -db = SQLAlchemy() - -# default environment is development, otherwie specified by .env -def create_app(environment='development'): - - # Get app type from .env - environment = os.environ.get('APP_ENVIRONMENT', environment) - - # Initial app and configuration - app = Flask(__name__, instance_relative_config=True) - - # Upload config from config.py - if environment == 'development': CORS(app) - app.config.from_object(config[environment]) - - # Logging - file_handler = RotatingFileHandler('api.log', maxBytes=10000, backupCount=1) - file_handler.setLevel(logging.INFO) - app.logger.addHandler(file_handler) - stdout = logging.StreamHandler(sys.stdout) - stdout.setLevel(logging.DEBUG) - app.logger.addHandler(stdout) - - # Initialize the application - db.init_app(app) - - # Add a command line interface to the application - app.cli.add_command(init_db_command) - - # The nscope endpoint application - from flaskr import nscope - - # The executor and blueprint are specified via nscope - nscope.executor.init_app(app) - app.register_blueprint(nscope.bp) - - # The primary application - return app - - -def init_db(): - db.drop_all() - db.create_all() - -@click.command("init-db") -@with_appcontext -def init_db_command(): - init_db() - click.echo("Initialized Database") - diff --git a/flaskr/auth/views.py b/flaskr/auth/views.py deleted file mode 100644 index 07a9d88..0000000 --- a/flaskr/auth/views.py +++ /dev/null @@ -1,121 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- -# vim:fenc=utf-8 -# -# Copyright © 2020 theo -# -# Distributed under terms of the MIT license. - -""" -Views for authentication prefixes -""" - -import functools - -from flask import Blueprint -from flask import flash -from flask import g -from flask import redirect -from flask import render_template -from flask import request -from flask import session -from flask import url_for - -from flaskr import db -from flaskr.auth.models import User - -bp = Blueprint("auth", __name__, url_prefix="/auth") - - -def login_required(view): - """View decorator that redirects anonymous users to the login page.""" - - @functools.wraps(view) - def wrapped_view(**kwargs): - if g.user is None: - return redirect(url_for("auth.login")) - - return view(**kwargs) - - return wrapped_view - - -@bp.before_app_request -def load_logged_in_user(): - """If a user id is stored in the session, load the user object from - the database into ``g.user``.""" - user_id = session.get("user_id") - g.user = User.query.get(user_id) if user_id is not None else None - - -@bp.route("/register", methods=("GET", "POST")) -def register(): - """Register a new user. - Validates that the email is not already taken. Hashes the - password for security. - """ - if request.method == "POST": - email = request.form["email"] - first_name = request.form["first_name"] - last_name = request.form["last_name"] - password = request.form["password"] - confirm_password = request.form["confirm_password"] - - error = None - - if not first_name or not last_name: - error = "Name is required" - elif not email: - error = "Email is required." - elif not password: - error = "Password is required." - elif password != confirm_password: - error = "Passwords do not match" - elif db.session.query( - User.query.filter_by(email=email).exists()).scalar(): - error = f"Email {email} is already registered." - - if error is None: - print("SUccess") - # the name is available, create the user and go to the login page - db.session.add(User(first_name=first_name, last_name=last_name, email=email, password=password)) - db.session.commit() - return redirect(url_for("auth.login")) - - print("Error") - flash(error) - - return render_template("register.html") - - -@bp.route("/login", methods=("GET", "POST")) -def login(): - """Log in a registered user by adding the user id to the session.""" - if request.method == "POST": - email = request.form["email"] - password = request.form["password"] - - error = None - user = User.query.filter_by(email=email).first() - - if user is None: - error = "Incorrect email." - elif not user.check_password(password): - error = "Incorrect password." - - if error is None: - # store the user id in a new session and return to the index - session.clear() - session["user_id"] = user.id - return redirect(url_for("nscope.index")) - - flash(error) - - return render_template("login.html") - - -@bp.route("/logout") -def logout(): - """Clear the current session, including the stored user id.""" - session.clear() - return redirect(url_for("nscope.index")) diff --git a/flaskr/config.py b/flaskr/config.py deleted file mode 100644 index 83e8952..0000000 --- a/flaskr/config.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -Config for flask app -""" - -import os -from dotenv import load_dotenv -load_dotenv() - - -POSTGRES = { - 'user': os.getenv('POSTGRES_USER', 'postgres'), - 'pw': os.getenv('POSTGRES_PASSWORD', 'root'), - 'db': os.getenv('POSTGRES_DB', 'postgres'), - 'host': os.getenv('POSTGRES_HOST', 'localhost'), - 'port': os.getenv('POSTGRES_PORT', 5432), -} - -TEST_POSTGRES = { - 'user': os.getenv('POSTGRES_USER', 'postgres'), - 'pw': os.getenv('POSTGRES_PASSWORD', 'root'), - 'db': os.getenv('POSTGRES_DB', 'postgres'), - 'host': os.getenv('POSTGRES_HOST', 'localhost'), - 'port': os.getenv('POSTGRES_PORT', 5432), -} - - -class Config: - ERROR_404_HELP = False - - SECRET_KEY = os.getenv('APP_SECRET', 'secret key') - - SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://{user}:{pw}@{host}:{port}/{db}'.format(**POSTGRES) - SQLALCHEMY_TRACK_MODIFICATIONS = False - - DOC_USERNAME = 'api' - DOC_PASSWORD = 'password' - - -class DevConfig(Config): - DEBUG = True - - -class TestConfig(Config): - SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://{user}:{pw}@{host}:{port}/{db}'.format(**TEST_POSTGRES) - TESTING = True - DEBUG = True - - -class ProdConfig(Config): - DEBUG = False - - -config = { - 'development': DevConfig, - 'testing': TestConfig, - 'production': ProdConfig -} diff --git a/flaskr/nscope/__init__.py b/flaskr/nscope/__init__.py deleted file mode 100644 index fd3c41d..0000000 --- a/flaskr/nscope/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -""" -Init file for nscope -""" - -from .views import bp, executor diff --git a/flaskr/nscope/math_lib/__init__.py b/flaskr/nscope/math_lib/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/flaskr/nscope/models.py b/flaskr/nscope/models.py deleted file mode 100644 index d1302bb..0000000 --- a/flaskr/nscope/models.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -Models Example -""" - -from sqlalchemy.ext.hybrid import hybrid_property -from werkzeug.security import check_password_hash -from werkzeug.security import generate_password_hash - -from flaskr import db - - -class User(db.Model): - id = db.Column(db.Integer, primary_key=True) - first_name = db.Column(db.String, unique=False, nullable=False) - last_name = db.Column(db.String, unique=False, nullable=False) - email = db.Column(db.String, unique=True, nullable=False) - _password = db.Column("password", db.String, nullable=False) - - @hybrid_property - def password(self): - return self._password - - @password.setter - def password(self, value): - """Store the password as a hash for security.""" - self._password = generate_password_hash(value) - - def check_password(self, value): - return check_password_hash(self.password, value) - -class Sequence(db.Model): - __tablename__ = 'sequences' - - id = db.Column(db.String, unique=True, nullable=False, primary_key=True) - name = db.Column(db.String, unique=False, nullable=True) - # The following is called the "offset" in the OEIS, but that is a - # Postgres reserved word, so we use a different name. - shift = db.Column(db.Integer, unique=False, nullable=False, default=0) - values = db.Column(db.ARRAY(db.String), unique=False, nullable=True) - values_requested = db.Column(db.Boolean, nullable=False, default=False) - raw_refs = db.Column(db.String, unique=False, nullable=True) - backrefs = db.Column(db.ARRAY(db.String), unique=False, nullable=True) - meta_requested = db.Column(db.Boolean, nullable=False, default=False) - # Sadly, multidimensional arrays can't vary in dimension - # so we store factorization arrays as strings - factors = db.Column(db.ARRAY(db.String), unique=False, nullable=True) - - @classmethod - def get_seq_by_id(self, id): - ret = self.query.filter_by(id=id).first() - return ret - - - - - - - diff --git a/flaskr/nscope/views.py b/flaskr/nscope/views.py deleted file mode 100644 index d0e6235..0000000 --- a/flaskr/nscope/views.py +++ /dev/null @@ -1,313 +0,0 @@ -""" -Views for nscope model -""" - -from flask import Blueprint, jsonify, render_template -from flask_executor import Executor -from flaskr import db -from flaskr.nscope.models import * - -import cypari2 -from cypari2.convert import gen_to_python -import re -import requests - - -executor = Executor() -bp = Blueprint("nscope", __name__) - - -# Creating a simple index route (this will error because we currently dont have an index.html"j -@bp.route("/index") -def index(): - return render_template("index.html") - -def fetch_metadata(oeis_id): - """ When called with a *valid* oeis id, makes sure the metadata has been - obtained, and returns the corresponding Sequence object with valid - metadata. - - Note that this also crawls all backreferences, so it can take quite - a long time for popular sequences (potentially hours). - """ - seq = find_oeis_sequence(oeis_id) - if seq.meta_requested: - if seq.raw_refs is None: - return LookupError(f"Metadata fetching for {oeis_id} in progress") - return seq - seq.meta_requested = True - db.session.commit() - # Now grab the data - match_url = f"https://oeis.org/search?q={seq.id}&fmt=json" - r = requests.get(match_url).json() - if r['results'] != None: # Found some metadata - backrefs = [] - target_number = int(seq.id[1:]) - matches = r['count'] - saw = 0 - while (saw < matches): - for result in r['results']: - if result['number'] == target_number: - seq.name = result['name'] - seq.raw_refs = "\n".join(result.get('xref', [])) - else: - backrefs.append('A' + str(result['number']).zfill(6)) - saw += 1 - if saw < matches: - r = requests.get(match_url + f"&start={saw}").json() - if r['results'] == None: - break - seq.backrefs = backrefs - db.session.commit() - return seq - -def find_oeis_sequence(oeis_id): - """ Returns a Sequence object associated with the given valid OEIS ID. - Only call this with a non-Exception return value of - get_valid_oeis_id(). - - If the oeis_id is not yet in the database, simply creates - a dummy Sequence entry and adds it to the database with no data. - So the returned Sequence object may not have any data. - - Note further that just finding a sequence does not schedule any - filling in of its data. That needs to be done judiciously by the - requests. - """ - seq = Sequence.get_seq_by_id(oeis_id) - if seq: return seq - # Note the sequence index might not correspond to an existing sequence - # but we just ignore that issue for the sake of returning quickly - seq = Sequence(id=oeis_id) - db.session.add(seq) - db.session.commit() - return seq - -def placeholder_name(oeis_id): - return f"{oeis_id} [name not yet loaded]" - -domain = 'https://oeis.org/' - -def fetch_values(oeis_id): - """ - When called with a valid oeis id, fetches the b-file from the - OEIS (if it has not already been), and returns a Sequence object - with the values filled in. - """ - # First check if it is in the database - seq = find_oeis_sequence(oeis_id) - # See if we already have the values: - if seq.values is not None: return seq - # See if getting them is in progress: - if seq.values_requested: - return LookupError("Value fetching for {oeis_id} in progress.") - seq.values_requested = True - db.session.commit() - # Now try to get it from the OEIS: - r = requests.get(f"{domain}{oeis_id}/b{oeis_id[1:]}.txt") - if r.status_code == 404: - return LookupError(f"B-file for ID '{oeis_id}' not found in OEIS.") - # Parse the b-file: - first = float('inf') - last = float('-inf') - name = '' - seq_vals = {} - for line in r.text.split("\n"): - if not line: continue - if line[0] == '#': - # Some sequences have info in first comment that we can use as a - # stopgap until the real name is obtained. - if not name: name = line[1:] - continue - column = line.split() - if len(column) < 2: continue - if not column[0][0].isdigit(): - return LookupError( - f"Unparseable b-file line for ID '{oeis_id}': {line}") - index = int(column[0]) - if index < first: first = index - if index > last: last = index - seq_vals[index] = column[1] - if last < first: - return IndexError(f"No terms found for ID '{oeis_id}'.") - seq.values = [seq_vals[i] for i in range(first,last+1)] - if not seq.name: - seq.name = name or placeholder_name(oeis_id) - db.session.commit() - return seq - -def fetch_factors(oeis_id, num_elements = -1): - """ The first argument oeis_id must be a valid OEIS id that is already - stored in the database **with all of its values**. - The second argument num_elements gives the number of terms to factor, - or the default -1 means to factor all known elements. - - This function factors the first num_elements terms (if they aren't - already) and adds them to the database. - Returns seq object if requested factors existed or were added - to the table; otherwise returns an error. - Note it _returns_ the Error object, rather than throwing it. - It will return the minimum of the number of requested factors - or the number of terms available from OEIS. - Terms too big to factor will store a factorization of 'no_fac'. - The factoring format otherwise is essentially that of pari, - stored as a string (since flask doesn't allow multidimensional - arrays with varying sizes). - """ - seq = find_oeis_sequence(oeis_id) - - if num_elements < 0 or len(seq.values) < num_elements: - num_elements = len(seq.values) - # Load from database how much has been factored already - if not seq.factors: - factors = [] - else: - # this copy appears to be required for the sequence update to work - factors = seq.factors.copy() - len_factors = len(factors) - if len_factors >= num_elements: - return seq - # Factor whatever else is requested, within reason. - pari = cypari2.Pari() - for i in range(len_factors, num_elements): - val = int(seq.values[i]) - # the factorization of 1 is empty - if val == 1: - fac = [] - elif abs(val) <= 2**200: # Arbitrary limit; a timeout would be better - fac = [] - # elements are arrays [p, e] for factor p^e - # including [-1,1] for negative numbers - # and [0,1] for zero - fac = gen_to_python(pari(val).factor()) - else: - fac = 'no_fac' - factors.append(str(fac).replace(" ","")); - # And further it seems that we are obliged to actually modify the identity - # of seq.factors in order for the database to update. It is hard to believe - # that both the .copy() above and this copy() are required, yet testing - # appeared to confirm that. - seq.factors = factors.copy() - db.session.commit() - return seq - -def fetch_values_and_factors(oeis_id): - """ Convenience sequencing function for get_oeis_metadata """ - seq = fetch_values(oeis_id) - if isinstance(seq, Exception): return seq - return fetch_factors(oeis_id) - -# The following regexp encodes the format of valid OEIS IDs. NOTE: when -# that format eventually changes, this code will have to be updated. -oeis_validator = re.compile(r'^A\d{6}$') -oeis_valid_format = 'Annnnnn' - -def get_valid_oeis_id(oeis_id): - """ Takes a string and returns either the associated valid OEIS id - or an Exception indicating the difficulty with the input id. - Note it does not raise the exception, just returns it. - """ - - if not isinstance(oeis_id, str): - return TypeError('Supplied oeis_id is not a string') - if len(oeis_id) != 7: - return SyntaxError(f"ID {oeis_id} is not 7 characters long") - valid_id = oeis_id - first_character = oeis_id[0] - if first_character.islower(): - """ If we can configure logging levels, e.g. info, warn, - error, debug, verbose, etc., then the following print - statements should be (verbose?) logs. - - TODO: - https://github.com/numberscope/backscope/issues/57 - """ - print('verbose: first character in oeis_id is lowercase') - print('verbose: making first character in oeis_id uppercase') - valid_id = first_character.upper() + valid_id[1:] - # The normal case: - if oeis_validator.match(valid_id): return valid_id - # Report appropriate error: - if oeis_id == valid_id: - return SyntaxError(f"ID {oeis_id} not of form {oeis_valid_format}") - return SyntaxError( - f"Neither {oeis_id} nor {valid_id} of form {oeis_valid_format}") - -@bp.route("/api/get_oeis_values//", methods=["GET"]) -def get_oeis_values(oeis_id, num_elements): - valid_oeis_id = get_valid_oeis_id(oeis_id) - if isinstance(valid_oeis_id, Exception): - return f"Error: {valid_oeis_id}" - seq = fetch_values(valid_oeis_id) - if isinstance(seq, Exception): - return f"Error: {seq}" - # OK, got valid sequence, so schedule grabbing of metadata and factors: - executor.submit(fetch_metadata, valid_oeis_id) - executor.submit(fetch_factors, valid_oeis_id) - # Finally, trim return sequence as requested: - raw_vals = seq.values - wants = int(num_elements) - if wants and wants < len(raw_vals): - raw_vals = raw_vals[0:wants] - vals = {(i+seq.shift):raw_vals[i] for i in range(len(raw_vals))} - - return jsonify({'id': seq.id, 'name': seq.name, 'values': vals}) - -@bp.route("/api/get_oeis_name_and_values/", methods=["GET"]) -def get_oeis_name_and_values(oeis_id): - valid_oeis_id = get_valid_oeis_id(oeis_id) - if isinstance(valid_oeis_id, Exception): - return f"Error: {valid_oeis_id}" - seq = fetch_values(valid_oeis_id) - if isinstance(seq, Exception): - return f"Error: {seq}" - raw_vals = seq.values - vals = {(i + seq.shift): raw_vals[i] for i in range(len(raw_vals))} - # Now get the name - seq = find_oeis_sequence(valid_oeis_id) - if not seq.name or seq.name == placeholder_name(oeis_id): - r = requests.get(f"{domain}search?q=id:{oeis_id}&fmt=json").json() - if r['results'] != None: - seq.name = r['results'][0]['name'] - db.session.commit() - executor.submit(fetch_factors, valid_oeis_id) - return jsonify({'id': seq.id, 'name': seq.name, 'values': vals}) - -@bp.route("/api/get_oeis_metadata/", methods=["GET"]) -def get_oeis_metadata(oeis_id): - valid_oeis_id = get_valid_oeis_id(oeis_id) - if isinstance(valid_oeis_id, Exception): - return f"Error: {valid_oeis_id}" - seq = fetch_metadata(valid_oeis_id) - if isinstance(seq, Exception): - return f"Error: {seq}" - executor.submit(fetch_values_and_factors, valid_oeis_id) - return jsonify({ - 'id': seq.id, - 'name': seq.name, - 'xrefs': seq.raw_refs, - 'backrefs': seq.backrefs - }) - -@bp.route("/api/get_oeis_factors//", methods=["GET"]) -def get_oeis_factors(oeis_id, num_elements): - valid_oeis_id = get_valid_oeis_id(oeis_id) - if isinstance(valid_oeis_id, Exception): - return f"Error: {valid_oeis_id}" - raw_vals = fetch_values(valid_oeis_id) - if isinstance(raw_vals, Exception): - return f"Error: {raw_vals}" - wants = int(num_elements) - seq = fetch_factors(valid_oeis_id, wants) - if isinstance(seq, Exception): - return f"Error: {seq}" - raw_fac = seq.factors - if wants and wants < len(raw_fac): - raw_fac = raw_fac[0:wants] - facs = {(i+seq.shift):raw_fac[i] for i in range(len(raw_fac))} - executor.submit(fetch_metadata, valid_oeis_id) - return jsonify({ - 'id': seq.id, - 'name': seq.name, - 'factors': facs - }) diff --git a/manage.py b/manage.py deleted file mode 100644 index 5c753a6..0000000 --- a/manage.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Manages Resources and runs server / database -""" - -import os -import sys - -from flask_script import Manager -from flask_migrate import Migrate, MigrateCommand - -from flaskr import create_app, db - -app = create_app() - -migrate = Migrate(app, db) -manager = Manager(app) - -manager.add_command('db', MigrateCommand) - -if __name__ == '__main__': - if sys.argv[1] == 'runserver': - print(''' - -Copyright 2020-2022 Regents of the University of Colorado. - -This project is licensed under the -[MIT License](https://opensource.org/licenses/MIT). See LICENSE.md. - - ''') - manager.run() - diff --git a/migrations/README b/migrations/README new file mode 100644 index 0000000..0e04844 --- /dev/null +++ b/migrations/README @@ -0,0 +1 @@ +Single-database configuration for Flask. diff --git a/migrations/alembic.ini b/migrations/alembic.ini new file mode 100644 index 0000000..ec9d45c --- /dev/null +++ b/migrations/alembic.ini @@ -0,0 +1,50 @@ +# A generic, single database configuration. + +[alembic] +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic,flask_migrate + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[logger_flask_migrate] +level = INFO +handlers = +qualname = flask_migrate + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..89f80b2 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,110 @@ +import logging +from logging.config import fileConfig + +from flask import current_app + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) +logger = logging.getLogger('alembic.env') + + +def get_engine(): + try: + # this works with Flask-SQLAlchemy<3 and Alchemical + return current_app.extensions['migrate'].db.get_engine() + except TypeError: + # this works with Flask-SQLAlchemy>=3 + return current_app.extensions['migrate'].db.engine + + +def get_engine_url(): + try: + return get_engine().url.render_as_string(hide_password=False).replace( + '%', '%%') + except AttributeError: + return str(get_engine().url).replace('%', '%%') + + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +config.set_main_option('sqlalchemy.url', get_engine_url()) +target_db = current_app.extensions['migrate'].db + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def get_metadata(): + if hasattr(target_db, 'metadatas'): + return target_db.metadatas[None] + return target_db.metadata + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=get_metadata(), literal_binds=True + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html + def process_revision_directives(context, revision, directives): + if getattr(config.cmd_opts, 'autogenerate', False): + script = directives[0] + if script.upgrade_ops.is_empty(): + directives[:] = [] + logger.info('No changes in schema detected.') + + connectable = get_engine() + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=get_metadata(), + process_revision_directives=process_revision_directives, + **current_app.extensions['migrate'].configure_args + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..2c01563 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/fc05a7fad53f_.py b/migrations/versions/fc05a7fad53f_.py new file mode 100644 index 0000000..9012c45 --- /dev/null +++ b/migrations/versions/fc05a7fad53f_.py @@ -0,0 +1,40 @@ +"""empty message + +Revision ID: fc05a7fad53f +Revises: +Create Date: 2023-05-24 11:31:21.698722 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'fc05a7fad53f' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('sequences', + sa.Column('oeis_id', sa.String(), nullable=False), + sa.Column('name', sa.String(), nullable=True), + sa.Column('oeis_offset', sa.Integer(), nullable=False), + sa.Column('vals', sa.ARRAY(sa.String()), nullable=True), + sa.Column('values_requested', sa.Boolean(), nullable=False), + sa.Column('raw_refs', sa.String(), nullable=True), + sa.Column('backrefs', sa.ARRAY(sa.String()), nullable=True), + sa.Column('metadata_requested', sa.Boolean(), nullable=False), + sa.Column('factors', sa.ARRAY(sa.String()), nullable=True), + sa.PrimaryKeyConstraint('oeis_id'), + sa.UniqueConstraint('oeis_id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('sequences') + # ### end Alembic commands ### diff --git a/requirements-freeze.txt b/requirements-freeze.txt new file mode 100644 index 0000000..07c01f4 --- /dev/null +++ b/requirements-freeze.txt @@ -0,0 +1,36 @@ +alembic==1.11.1 +asgiref==3.7.1 +black==23.3.0 +blinker==1.6.2 +certifi==2023.5.7 +charset-normalizer==3.1.0 +click==8.1.3 +cypari2==2.1.3 +cysignals==1.11.2 +Cython==0.29.34 +exceptiongroup==1.1.1 +Flask==2.3.2 +Flask-Migrate==4.0.4 +Flask-SQLAlchemy==3.0.3 +greenlet==2.0.2 +idna==3.4 +iniconfig==2.0.0 +itsdangerous==2.1.2 +Jinja2==3.1.2 +Mako==1.2.4 +MarkupSafe==2.1.2 +mypy-extensions==1.0.0 +packaging==23.1 +pathspec==0.11.1 +platformdirs==3.5.1 +pluggy==1.0.0 +psycopg2-binary==2.9.6 +pytest==7.3.1 +python-dotenv==1.0.0 +requests==2.31.0 +ruff==0.0.267 +SQLAlchemy==2.0.15 +tomli==2.0.1 +typing_extensions==4.6.1 +urllib3==2.0.2 +Werkzeug==2.3.4 diff --git a/requirements.txt b/requirements.txt index e8a5aee..6807356 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,32 +1,10 @@ -alembic==1.8.1 -certifi==2022.9.14 -charset-normalizer==2.1.1 -click==7.1.2 -cypari2==2.1.3 -cysignals==1.11.2 -Cython==0.29.33 -Flask==1.1.4 -flask-cors==3.0.10 -Flask-Executor==0.10.0 -Flask-Login==0.5.0 -Flask-Migrate==2.7.0 -Flask-Script==2.0.6 -Flask-SQLAlchemy==2.5.1 -Flask-WTF==0.15.1 -idna==3.4 -itsdangerous==1.1.0 -Jinja2==2.11.3 -jsonify==0.5 -Mako==1.2.1 -MarkupSafe==1.1.1 -numpy==1.23.2 -psycopg2-binary==2.9.3 -python-dateutil==2.8.2 -python-dotenv==0.20.0 -python-editor==1.0.4 -requests==2.28.1 -six==1.16.0 -SQLAlchemy==1.4.40 -urllib3==1.26.12 -Werkzeug==1.0.1 -WTForms==2.3.3 +Flask[async] # web framework with the async extra +black # code formatter +ruff # code linter +pytest # test framework +python-dotenv # library for .env files +Flask-SQLAlchemy # Flask library for a popular SQL ORM +psycopg2-binary # library for interacting with Postgres +Flask-Migrate # library for migrating database models +requests # library for making HTTP requests +cypari2 # library for factoring diff --git a/sequence.py b/sequence.py new file mode 100644 index 0000000..d508c41 --- /dev/null +++ b/sequence.py @@ -0,0 +1,312 @@ +""" +This module defines the Sequence class, and some functions that get data +for sequences. The Sequence class contains the database model for an +OEIS sequence. The Sequence class also defines a few methods. +""" + +import os +from flask_sqlalchemy import SQLAlchemy +from sqlalchemy.exc import NoResultFound +import requests +import re +import cypari2 +from cypari2.convert import gen_to_python + +db = SQLAlchemy() + + +class Sequence(db.Model): + """Define the DB model and methods for a sequence. + + This is class contains a model for the data we want to store in the + database about an OEIS sequence. We use an object relational mapper + called SQLAlchemy (the package is Flask SQLAlchemy) to describe the + data we want to store in the database, and it does the heavy lifting + of interacting with the database. Normally, you'd have to write SQL + queries to insert and extract data from the database, but ORMs + abstract that work away from the developer. + + This class also defines a few methods. + """ + + __tablename__ = "sequences" + + # TODO: Consider renaming OEIS ID to A-number: oeis_id->a_number. + # We at Numberscope seem to have taken to calling A-numbers OEIS + # IDs (because they function as IDs), but it might make sense to + # use OEIS terminology. This would require a database migration. + oeis_id = db.Column(db.String, unique=True, nullable=False, primary_key=True) + name = db.Column(db.String, unique=False, nullable=True) + oeis_offset = db.Column(db.Integer, unique=False, nullable=False, default=0) + vals = db.Column(db.ARRAY(db.String), unique=False, nullable=True) + values_requested = db.Column(db.Boolean, nullable=False, default=False) + raw_refs = db.Column(db.String, unique=False, nullable=True) + backrefs = db.Column(db.ARRAY(db.String), unique=False, nullable=True) + metadata_requested = db.Column(db.Boolean, nullable=False, default=False) + + # Sadly, multidimensional arrays can't vary in dimension, so we + # store factorization arrays as strings. + factors = db.Column(db.ARRAY(db.String), unique=False, nullable=True) + + def as_dict(self): + """Return a sequence object as a dictionary. Helper method. + + Taken from https://stackoverflow.com/a/11884806/15027348. + + :return: sequence object as a dictionary + """ + return {c.name: getattr(self, c.name) for c in self.__table__.columns} + + @classmethod + def get(cls, oeis_id): + """Try to get a sequence object from the database. + + :param oeis_id: OEIS sequence ID, typically Annnnnn where n is a number + :return: sequence object for given ID or None + """ + try: + valid_oeis_id = get_valid_oeis_id(oeis_id) + return db.session.execute( + db.select(Sequence).filter_by(oeis_id=valid_oeis_id) + ).scalar_one() + except NoResultFound: + return None + + +# The following regular expression encodes the format of valid OEIS IDs. +# When that format eventually changes, this code will have to be updated. +oeis_validator = re.compile(r"^A\d{6}$") +oeis_valid_format = "Annnnnn, where n is a number" + + +def get_valid_oeis_id(oeis_id): + """Return a capitalized, valid OEIS ID or raise a SyntaxError. + + :param oeis_id: invalid or valid OEIS sequence ID, possibly annnnnn + :return: valid OEIS sequence ID, typically Annnnnn where n is a number + """ + + # Capitalize the first character. + valid_id = oeis_id + first_character = oeis_id[0] + if first_character.islower(): + valid_id = first_character.upper() + valid_id[1:] + + # Check if the supplied ID matches the regular expression. + if oeis_validator.match(valid_id): + return valid_id + else: + raise SyntaxError(f"id {oeis_id} not of form {oeis_valid_format}") + + +def find_oeis_sequence(oeis_id): + """Get the sequence object associated with the given OEIS ID. + + If the object associated with the given OEIS ID isn't in the + database yet, this function adds a sequence to the database + with no data other than the ID filled in. + + Finding a sequence doesn't schedule any filling in of the data. + + :param oeis_id: OEIS sequence ID, typically Annnnnn where n is a number + :return: a sequence object, possibly one with just the ID filled in + """ + valid_oeis_id = get_valid_oeis_id(oeis_id) + seq = Sequence.get(valid_oeis_id) + if seq: + return seq + + # Note the sequence index might not correspond to an existing + # sequence, but we just ignore that issue for the sake of returning + # quickly. + seq = Sequence(oeis_id=valid_oeis_id) + db.session.add(seq) + db.session.commit() + return seq + + +def fetch_values(oeis_id): + """Make an HTTP request to get sequence values from the OEIS. + + An OEIS b-file is a text file that contains the elements of an + integer sequence. Each line of the text file has the index of + an element followed by a space, followed by the value of the + element at that index. + + :param oeis_id: OEIS sequence ID, typically Annnnnn where n is a number + :return: dict of index->value for given OEIS sequence + """ + + seq = find_oeis_sequence(oeis_id) + if seq.vals: + return {(i + seq.oeis_offset): seq.vals[i] for i in range(len(seq.vals))} + if seq.values_requested: + raise LookupError(f"value fetching for {oeis_id} in progress") + + response = requests.get(f"{os.environ.get('OEIS_URL')}{oeis_id}/b{oeis_id[1:]}.txt") + if response.status_code == 200: + # If there are too many requests being made to the OEIS, we + # might get a warning from the OEIS about crawling too fast. + # Presumably such a warning would not have a 200 status code. + # One would think the OEIS would use a 429 status code. If we + # get something other than 200, we don't want to say value + # fetching is in progress because the request might not have + # succeeded. + seq.values_requested = True + db.session.commit() + else: + raise requests.exceptions.RequestException( + f"response for {oeis_id} status: {response.status_code}" + ) + + vals = {} + + # Parse the b-file. + first = float("inf") + last = float("-inf") + for line in response.text.split("\n"): + # Sometimes lines are skipped or commented. + if not line or line[0] == "#": + continue + + # The first item in a line should be the index, followed + # by a space, followed by the value at that index. + index_and_value = line.split() + if len(index_and_value) < 2: + continue + + if not index_and_value[0][0].isdigit(): + raise LookupError( + f"unable to parse b-file line for id: {oeis_id}, line: {line}" + ) + + # Handle the fact that not all sequences have the same + # indexing scheme. + index = int(index_and_value[0]) + if index < first: + first = index + if index > last: + last = index + vals[index] = index_and_value[1] + + if last < first: + raise IndexError(f"no terms found for id: {oeis_id}") + + seq.vals = [vals[i] for i in range(first, last + 1)] + db.session.commit() + return vals + + +def fetch_metadata(oeis_id): + """Get sequence metadata from the OEIS. Might take long time. + + We can use the search route on the OEIS website with the query + parameter that specifies we want JSON to get metadata about + a sequence. The field in the JSON response we are most concerned + about is an array named results. This is the set of results that + you would see if you were searching the OEIS using the web + interface. + + This also crawls all backreferences, so it can take quite + a long time for popular sequences (potentially hours). + + :param oeis_id: OEIS sequence ID, typically Annnnnn where n is a number + :return: sequence object with populated metadata fields + """ + valid_oeis_id = get_valid_oeis_id(oeis_id) + seq = find_oeis_sequence(valid_oeis_id) + + # If we already got the metadata, return it. Otherwise, tell the + # caller that the fetch is in progress. + if seq.metadata_requested: + if seq.raw_refs is None: + raise LookupError(f"metadata fetching for {oeis_id} in progress") + return seq + + query_url = f"https://oeis.org/search?q={seq.oeis_id}&fmt=json" + response = requests.get(query_url).json() + seq.metadata_requested = True + db.session.commit() + + if response["results"] is not None: + backrefs = [] + + # Each results object has a number field, which is what we + # refer to as the OEIS ID. For instance, the number for OEIS + # ID A000001 is 1. The number for OEIS ID A12345 is 12345. + target_number = int(seq.oeis_id[1:]) + + # TODO: What is this count? + # It's not the number of results. For instance, the number of + # results for A000001 is 10, and the count for the A000001 query + # is 160. As of this writing, the wiki page for the JSON format + # doesn't say what the count is: + # https://oeis.org/wiki/JSON_Format. + matches = response["count"] + saw = 0 # TODO: What are we "seeing" here? + while saw < matches: + for result in response["results"]: + if result["number"] == target_number: + seq.name = result["name"] + seq.raw_refs = "\n".join(result["xref"]) + else: + # Concatenate the string "A" with the number with + # enough zeroes added at the beginning of it so that + # the number is six characters in length. Thus, we + # obtain what we call the OEIS ID. For instance, the + # number 1 is converted to A000001. + backrefs.append("A" + str(result["number"]).zfill(6)) + saw += 1 + + # TODO: Explain this block of code. + # I, Liam, am not sure what this block of code does. It + # seems like the above loop is going to exhaust the results + # for the given query, and I'm not sure what changing + # the start does to the JSON response. + if saw < matches: + response = requests.get(query_url + f"&start={saw}").json() + if response["results"] is None: + break + seq.backrefs = backrefs + db.session.commit() + return seq + + +def fetch_factors(oeis_id): + """Get the factors for each element in the sequence. + + :param oeis_id: OEIS sequence ID, typically Annnnnn where n is a number + :return: a list containing list of factors for each element in the sequence + """ + valid_oeis_id = get_valid_oeis_id(oeis_id) + fetch_values(valid_oeis_id) # Ensure the sequence object has values. + seq = find_oeis_sequence(valid_oeis_id) + factors = [] + if seq.vals: # After fetching values, this should be redundant. + pari = cypari2.Pari() + for str_val in seq.vals: + val = int(str_val) + + # 2^200 is an arbitrary limit; a timeout would be better. + if val == 1: + # The factorization of 1 is empty, so we leave 1's + # val_fac array as an empty array. + val_fac = [] + elif abs(val) <= 2**200: + # Elements are arrays [p, e] for factor p^e including + # [-1,1] for negative numbers and [0,1] for zero. + val_fac = gen_to_python(pari(val).factor()) + else: + val_fac = "no_fac" + factors.append(str(val_fac).replace(" ", "")) + + # It seems like we need the shallow copy of factors when setting + # a value for seq.factors. + seq.factors = factors.copy() + db.session.commit() + + # TODO: Return a dict with index+offset->factors. + # Once we actually get the offset for a sequence, we need to use it + # to create a dictionary where the keys are index+offset and the + # values are the factor arrays for index+offset. + return factors diff --git a/server/README.md b/server/README.md index 3fb7a57..5142685 100644 --- a/server/README.md +++ b/server/README.md @@ -1,4 +1,5 @@ -# Numberscope Server Configuration +Numberscope Server Configuration +================================ The files contained in this directory are configuration files for Numberscope's server. diff --git a/server/production.sh b/server/production.sh index 7115cb2..7a296e4 100755 --- a/server/production.sh +++ b/server/production.sh @@ -2,5 +2,5 @@ cd /home/scope/repos/backscope && source .venv/bin/activate && pip install -r requirements.txt && -pip install --force cypari2 && -gunicorn --workers 3 --bind unix:/home/scope/repos/backscope/backscope.sock -m 777 wsgi:app +pip install gunicorn +gunicorn --workers 4 --bind unix:/home/scope/repos/backscope/backscope.sock -m 777 'app:create_app()' diff --git a/test_app.py b/test_app.py new file mode 100644 index 0000000..901dee5 --- /dev/null +++ b/test_app.py @@ -0,0 +1,116 @@ +"""" +This module is responsible for testing backscope's API routes. +""" + +import json +import pytest +from app import create_app + + +@pytest.fixture() +def app(): + app = create_app() + app.config.update( + { + "TESTING": True, + } + ) + yield app + + +@pytest.fixture() +def client(app): + return app.test_client() + + +def test_root(client): + """Ensure we can get a "Hello, World!" via the API. (Super important!) + + :param client: Flask test client + """ + response = client.get("/") + assert b"

Hello, World!

" in response.data + + +def test_get_oeis_sequence(client): + """Ensure we can get a sequence object via the API. + + This test depends on the names of the Sequence class keys. If they + change, this test will need to change. + + This test also depends on the values of sequence A000001. If the + values change or if the OEIS ID changes, this test will need to + change. + + :param client: Flask test client + """ + response = client.get("/api/get_oeis_sequence/A000001") + seq_dict = json.loads(response.data) + + # Assert each key is present. + assert "oeis_id" in seq_dict + assert "name" in seq_dict + assert "oeis_offset" in seq_dict + assert "vals" in seq_dict + assert "values_requested" in seq_dict + assert "raw_refs" in seq_dict + assert "backrefs" in seq_dict + assert "metadata_requested" in seq_dict + + # Check to see if a value is present in the values list. Index 160 + # has the value 238 for A000001. + assert "238" in seq_dict["vals"] + + +def test_get_oeis_values(client): + """Ensure we can get values via the API. + + This test depends on the values of sequence A000001. If the values + change or if the OEIS ID changes, this test will need to change. + + :param client: Flask test client + """ + response = client.get("/api/get_oeis_values/A000001") + values_dict = json.loads(response.data) + + # Assert values has keys 0 to 2047. + for i in range(2048): + assert str(i) in values_dict + + # Check to see if a value is present. Index 64 has the value 267 for + # A000001. + assert values_dict["64"] == "267" + + +def test_get_oeis_metadata(client): + """Ensure we can get metadata via the API. + + This test depends on the values of sequence A000001. If the metadata + associated with A000001 changes, this test will need to change. + + :param client: Flask test client + """ + response = client.get("/api/get_oeis_metadata/A000001") + metadata_dict = json.loads(response.data) + + # Assert metadata keys are present. + assert "name" in metadata_dict + assert "xrefs" in metadata_dict + assert "backrefs" in metadata_dict + + # Assert some of the values are present. + assert metadata_dict["name"] == "Number of groups of order n." + assert "A000679" in metadata_dict["xrefs"] + assert "A003277" in metadata_dict["backrefs"] + + +def test_get_oeis_factors(client): + """Ensure we can get factors via the API. + + :param client: Flask test client + """ + response = client.get("/api/get_oeis_factors/A000001") + factors_list = json.loads(response.data) + + # TODO: Change this when we return a dict from get_oeis_factors. + assert factors_list[0] == "[[0,1]]" diff --git a/tools/pre-commit b/tools/pre-commit new file mode 100755 index 0000000..1bd0a1e --- /dev/null +++ b/tools/pre-commit @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# Format the code. +black . + +# Lint the code. +ruff check . + +# Test the code. +pytest \ No newline at end of file diff --git a/wsgi.py b/wsgi.py deleted file mode 100644 index dcc90c2..0000000 --- a/wsgi.py +++ /dev/null @@ -1,2 +0,0 @@ -from flaskr import create_app -app = create_app()