From 5a49aaf0290b6f703a2ed29b5aa143953c47f7a9 Mon Sep 17 00:00:00 2001 From: Eric Yoon Date: Fri, 1 Mar 2024 00:40:00 -0500 Subject: [PATCH 1/9] WIP on persistent table --- app/api.py | 2 +- app/models.py | 17 +++++++++++++++++ app/scraper/__init__.py | 9 ++++++--- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/app/api.py b/app/api.py index 816a1d7..a9fe7eb 100644 --- a/app/api.py +++ b/app/api.py @@ -3,7 +3,7 @@ import time from app import db, cas from app.util import to_json, fail, succ, requires_login -from app.models import User, Person, Group +from app.models import User, Person, Group, PersonPersistent api_bp = Blueprint('api', __name__) diff --git a/app/models.py b/app/models.py index 99e49b1..dda00ff 100644 --- a/app/models.py +++ b/app/models.py @@ -199,6 +199,23 @@ def search(criteria): people = person_query.all() return people +class PersonPersistent(db.Model): + __tablename__ = 'person_persistent' + id = db.Column(db.Integer, primary_key=True, autoincrement=True) + person_id = db.Column(db.Integer, db.ForeignKey('person.id')) + + # socials + socials_instagram = db.Column(db.String) + socials_snapchat = db.Column(db.String) + + # privacy + privacy_hide_image = db.Column(db.Boolean) + privacy_hide_email = db.Column(db.Boolean) + privacy_hide_room = db.Column(db.Boolean) + privacy_hide_phone = db.Column(db.Boolean) + privacy_hide_address = db.Column(db.Boolean) + privacy_hide_major = db.Column(db.Boolean) + privacy_hide_birthday = db.Column(db.Boolean) class Group(db.Model): __tablename__ = 'group' diff --git a/app/scraper/__init__.py b/app/scraper/__init__.py index 74f245e..f677c87 100644 --- a/app/scraper/__init__.py +++ b/app/scraper/__init__.py @@ -1,5 +1,5 @@ from app import app, db, celery, elasticsearch -from app.models import leaderships, Group, Person +from app.models import leaderships, Group, Person, PersonPersistent from app.mail import send_scraper_report from app.scraper import sources @@ -106,8 +106,11 @@ def scrape(caches_active, face_book_cookie, people_search_session_cookie, csrf_t Group.query.delete() Person.query.delete() - elasticsearch.indices.delete(index=Person.__tablename__) - elasticsearch.indices.create(index=Person.__tablename__) + + if elasticsearch is not None: + elasticsearch.indices.delete(index=Person.__tablename__) + elasticsearch.indices.create(index=Person.__tablename__) + num_inserted = 0 for person_dict in people: if not person_dict.get('netid'): From e0988748cc1fee61fc50a054ec3bafaac02603b7 Mon Sep 17 00:00:00 2001 From: Eric Yoon Date: Mon, 18 Mar 2024 15:13:29 -0700 Subject: [PATCH 2/9] Endpoints for editing --- app/routes.py | 27 +++++++++++++++++++++------ app/templates/edit.html | 0 app/util.py | 8 ++++++++ 3 files changed, 29 insertions(+), 6 deletions(-) create mode 100644 app/templates/edit.html diff --git a/app/routes.py b/app/routes.py index 7c27af3..fceee6b 100644 --- a/app/routes.py +++ b/app/routes.py @@ -1,7 +1,7 @@ from flask import render_template, make_response, request, redirect, url_for, jsonify, abort, g, session from app import app, db, scraper, cas -from app.models import User, Person, Key -from app.util import requires_login, to_json, get_now, succ, fail +from app.models import User, Person, Key, PersonPersistent +from app.util import requires_login, forbidden_via_api, to_json, get_now, succ, fail from .cas_validate import validate from sqlalchemy import distinct @@ -25,12 +25,12 @@ def store_user(): token_cookie = request.cookies.get('token') if token_cookie: g.user = User.from_token(token_cookie) - method_used = 'cookie' + g.method_used = 'cookie' authorization = request.headers.get('Authorization') if g.user is None and authorization: token = authorization.split(' ')[-1] g.user = User.from_token(token) - method_used = 'header' + g.method_used = 'header' if g.user is None and cas.username: g.user = User.query.get(cas.username) if not g.user: @@ -41,14 +41,14 @@ def store_user(): registered_on=timestamp, admin=is_first_user) db.session.add(g.user) - method_used = 'CAS' + g.method_used = 'CAS' if g.user: g.person = Person.query.filter_by(netid=g.user.id, school_code='YC').first() if g.user.banned or (not g.person and not g.user.admin): # TODO: give a more graceful error than just a 403 abort(403) try: - print(f'Authorized request by {g.person.first_name} {g.person.last_name} with {method_used} authentication.') + print(f'Authorized request by {g.person.first_name} {g.person.last_name} with {g.method_used} authentication.') except AttributeError: print('Could not render name.') g.user.last_seen = timestamp @@ -271,6 +271,21 @@ def delete_key(key_id): db.session.commit() return succ('Key deleted.') +@app.route('/edit', methods=['GET']) +@requires_login +def edit(): + return render_template('edit.html') + +@app.route('/edit', methods=['POST']) +@requires_login +# @forbidden_via_api +def edit_post(): + payload = request.get_json() + if g.person is None: + return fail('Could not find person in the database.', 403) + print('Editing person! ', g.person.netid) + return succ('Person edited.') + @app.route('/authorize', methods=['POST']) def api_authorize_cas(): diff --git a/app/templates/edit.html b/app/templates/edit.html new file mode 100644 index 0000000..e69de29 diff --git a/app/util.py b/app/util.py index adbaeaa..91ad4c1 100644 --- a/app/util.py +++ b/app/util.py @@ -15,6 +15,14 @@ def wrapper_requires_login(*args, **kwargs): return wrapper_requires_login +def forbidden_via_api(f): + @wraps(f) + def wrapper_forbidden_via_api(*args, **kwargs): + if g.method_used == 'header': + return fail('This endpoint is not accessible via the API.', 403) + return f(*args, **kwargs) + return wrapper_forbidden_via_api + def succ(message, code=200): return ( jsonify({ From c7902852eac99c2983c8d0eea0009d8b62617b23 Mon Sep 17 00:00:00 2001 From: Eric Yoon Date: Mon, 18 Mar 2024 15:22:12 -0700 Subject: [PATCH 3/9] Migration file --- app/models.py | 1 - app/routes.py | 12 ++---- .../versions/f670f536d349_add_persistent.py | 42 +++++++++++++++++++ 3 files changed, 45 insertions(+), 10 deletions(-) create mode 100644 migrations/versions/f670f536d349_add_persistent.py diff --git a/app/models.py b/app/models.py index dda00ff..bdb9b79 100644 --- a/app/models.py +++ b/app/models.py @@ -216,7 +216,6 @@ class PersonPersistent(db.Model): privacy_hide_address = db.Column(db.Boolean) privacy_hide_major = db.Column(db.Boolean) privacy_hide_birthday = db.Column(db.Boolean) - class Group(db.Model): __tablename__ = 'group' __searchable__ = ( diff --git a/app/routes.py b/app/routes.py index fceee6b..1bf78dd 100644 --- a/app/routes.py +++ b/app/routes.py @@ -237,6 +237,7 @@ def hide_me(): @app.route('/keys', methods=['GET']) @requires_login +@forbidden_via_api def get_keys(): keys = Key.query.filter_by(user_id=g.user.id, deleted=False).all() @@ -245,6 +246,7 @@ def get_keys(): @app.route('/keys', methods=['POST']) @requires_login +@forbidden_via_api def create_key(): payload = request.get_json() key = g.user.create_key(payload['description']) @@ -252,17 +254,9 @@ def create_key(): db.session.commit() return to_json(key) - -""" -@app.route('/keys/', methods=['POST']) -@requires_login -def update_key(key_id): - pass -""" - - @app.route('/keys/', methods=['DELETE']) @requires_login +@forbidden_via_api def delete_key(key_id): key = Key.query.get(key_id) if key.user_id != g.user.id: diff --git a/migrations/versions/f670f536d349_add_persistent.py b/migrations/versions/f670f536d349_add_persistent.py new file mode 100644 index 0000000..411d17f --- /dev/null +++ b/migrations/versions/f670f536d349_add_persistent.py @@ -0,0 +1,42 @@ +"""add_persistent + +Revision ID: f670f536d349 +Revises: 31fcfd2925ea +Create Date: 2024-03-18 15:21:14.671517 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'f670f536d349' +down_revision = '31fcfd2925ea' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('person_persistent', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('person_id', sa.Integer(), nullable=True), + sa.Column('socials_instagram', sa.String(), nullable=True), + sa.Column('socials_snapchat', sa.String(), nullable=True), + sa.Column('privacy_hide_image', sa.Boolean(), nullable=True), + sa.Column('privacy_hide_email', sa.Boolean(), nullable=True), + sa.Column('privacy_hide_room', sa.Boolean(), nullable=True), + sa.Column('privacy_hide_phone', sa.Boolean(), nullable=True), + sa.Column('privacy_hide_address', sa.Boolean(), nullable=True), + sa.Column('privacy_hide_major', sa.Boolean(), nullable=True), + sa.Column('privacy_hide_birthday', sa.Boolean(), nullable=True), + sa.ForeignKeyConstraint(['person_id'], ['person.id'], ), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('person_persistent') + # ### end Alembic commands ### From cc68eeced0655ffc5c193c7bd469de58122a1e1b Mon Sep 17 00:00:00 2001 From: Eric Yoon Date: Mon, 18 Mar 2024 15:49:59 -0700 Subject: [PATCH 4/9] Edit route working --- app/models.py | 3 +++ app/routes.py | 23 +++++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/app/models.py b/app/models.py index bdb9b79..94deb02 100644 --- a/app/models.py +++ b/app/models.py @@ -199,6 +199,9 @@ def search(criteria): people = person_query.all() return people + def get_persistent_data(self): + return PersonPersistent.query.filter_by(person_id=self.id).first() + class PersonPersistent(db.Model): __tablename__ = 'person_persistent' id = db.Column(db.Integer, primary_key=True, autoincrement=True) diff --git a/app/routes.py b/app/routes.py index 1bf78dd..26dfca2 100644 --- a/app/routes.py +++ b/app/routes.py @@ -277,9 +277,28 @@ def edit_post(): payload = request.get_json() if g.person is None: return fail('Could not find person in the database.', 403) - print('Editing person! ', g.person.netid) - return succ('Person edited.') + + person_persistent = g.person.get_persistent_data() + if person_persistent is None: + person_persistent = PersonPersistent(person_id=g.person.id) + db.session.add(person_persistent) + + for key in [ + 'socials_instagram', + 'socials_snapchat', + 'privacy_hide_image', + 'privacy_hide_email', + 'privacy_hide_room', + 'privacy_hide_phone', + 'privacy_hide_address', + 'privacy_hide_major', + 'privacy_hide_birthday' + ]: + if key in payload: + setattr(person_persistent, key, payload[key]) + db.session.commit() + return succ('Person edited.') @app.route('/authorize', methods=['POST']) def api_authorize_cas(): From 2922f5aad1f8cb132f5efefe9eced0b86d2c860d Mon Sep 17 00:00:00 2001 From: Eric Yoon Date: Mon, 18 Mar 2024 16:26:49 -0700 Subject: [PATCH 5/9] Edit page working --- app/routes.py | 22 ++++++++++++++--- app/static/css/edit.css | 13 ++++++++++ app/static/js/edit.js | 38 ++++++++++++++++++++++++++++ app/templates/base.html | 1 + app/templates/edit.html | 55 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 126 insertions(+), 3 deletions(-) create mode 100644 app/static/css/edit.css create mode 100644 app/static/js/edit.js diff --git a/app/routes.py b/app/routes.py index 26dfca2..a6443bc 100644 --- a/app/routes.py +++ b/app/routes.py @@ -268,16 +268,32 @@ def delete_key(key_id): @app.route('/edit', methods=['GET']) @requires_login def edit(): - return render_template('edit.html') + if g.person is None: + return fail('Could not find person in the database.', 403) + person_persistent = g.person.get_persistent_data() + + if person_persistent is None: + return render_template('edit.html') + return render_template( + 'edit.html', + socials_instagram=person_persistent.socials_instagram, + socials_snapchat=person_persistent.socials_snapchat, + privacy_hide_image=person_persistent.privacy_hide_image, + privacy_hide_email=person_persistent.privacy_hide_email, + privacy_hide_room=person_persistent.privacy_hide_room, + privacy_hide_phone=person_persistent.privacy_hide_phone, + privacy_hide_address=person_persistent.privacy_hide_address, + privacy_hide_major=person_persistent.privacy_hide_major, + privacy_hide_birthday=person_persistent.privacy_hide_birthday + ) @app.route('/edit', methods=['POST']) @requires_login -# @forbidden_via_api +@forbidden_via_api def edit_post(): payload = request.get_json() if g.person is None: return fail('Could not find person in the database.', 403) - person_persistent = g.person.get_persistent_data() if person_persistent is None: person_persistent = PersonPersistent(person_id=g.person.id) diff --git a/app/static/css/edit.css b/app/static/css/edit.css new file mode 100644 index 0000000..fc06279 --- /dev/null +++ b/app/static/css/edit.css @@ -0,0 +1,13 @@ +.form-group { + display: flex; + flex-direction: row; + justify-content: flex-start; + align-items: center; + gap: 20px; + margin-bottom: 10px; +} + +h3 { + margin-bottom: 20px; + font-size: 20px; +} \ No newline at end of file diff --git a/app/static/js/edit.js b/app/static/js/edit.js new file mode 100644 index 0000000..ec6859e --- /dev/null +++ b/app/static/js/edit.js @@ -0,0 +1,38 @@ +const FIELDS = [ + 'socials_instagram', + 'socials_snapchat', + 'privacy_hide_image', + 'privacy_hide_email', + 'privacy_hide_room', + 'privacy_hide_phone', + 'privacy_hide_address', + 'privacy_hide_major', + 'privacy_hide_birthday' +]; + +const submit = document.getElementById('submit'); +submit.onclick = async (e) => { + const data = {}; + for(const field of FIELDS) { + const elem = document.getElementById(field); + if(elem.type === 'checkbox') data[field] = elem.checked; + else if(elem.type === 'text') data[field] = elem.value; + } + let response; + try { + response = await fetch('/edit', { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(data) + }) + } catch(e) { + console.error(e); + } + if(response.status === 200) { + window.location.reload(); + } else { + console.error(response); + } +} diff --git a/app/templates/base.html b/app/templates/base.html index 8d4f7f8..6ff2d87 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -21,6 +21,7 @@ + diff --git a/app/templates/edit.html b/app/templates/edit.html index e69de29..23bc15b 100644 --- a/app/templates/edit.html +++ b/app/templates/edit.html @@ -0,0 +1,55 @@ +{% extends "base.html" %} + +{% block content %} +

Edit Profile

+ +

Socials

+

You can list your social media profiles here. This information will be accessible to other Yale students.

+
+ + +
+
+ + +
+ +

Privacy

+

You can choose to hide certain info on Yalies.io.

+

+ Important: even if you hide your info here, it will still be visible via official Yale websites. + See FAQ for instructions on how to hide your information globally. +

+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ + + + +{% endblock %} From 012e97eda52222ee0540d89c512304908bb04743 Mon Sep 17 00:00:00 2001 From: Eric Yoon Date: Mon, 18 Mar 2024 16:53:32 -0700 Subject: [PATCH 6/9] WIP --- app/api.py | 3 ++- app/models.py | 25 ++++++++++++++++++++++++- app/routes.py | 14 ++------------ app/util.py | 12 ++++++++++++ 4 files changed, 40 insertions(+), 14 deletions(-) diff --git a/app/api.py b/app/api.py index a9fe7eb..7f03fce 100644 --- a/app/api.py +++ b/app/api.py @@ -50,7 +50,7 @@ def api_students(): if not criteria['filters'].get('school_code'): criteria['filters']['school_code'] = [] criteria['filters']['school_code'].append('YC') - students = Person.search(criteria) + students = Person.search(criteria) # TODO: Change this to respect privacy return to_json(students) @@ -62,6 +62,7 @@ def api_people(): except: criteria = {} people = Person.search(criteria) + # people = Person.search_respect_privacy_include_persistent(criteria) return to_json(people) diff --git a/app/models.py b/app/models.py index 94deb02..958bc5c 100644 --- a/app/models.py +++ b/app/models.py @@ -1,9 +1,11 @@ from app import app, db from app.search import SearchableMixin -from app.util import get_now +from app.util import get_now, PERSISTENT_FIELDS import jwt import datetime +from copy import copy from sqlalchemy.sql import collate +from sqlalchemy.orm.session import make_transient leaderships = db.Table( @@ -202,6 +204,27 @@ def search(criteria): def get_persistent_data(self): return PersonPersistent.query.filter_by(person_id=self.id).first() + @staticmethod + def search_respect_privacy_include_persistent(criteria): + ''' + Performs a search, including persistent data. + - Persistent data, including privacy and social information, is returned. + - If a person's privacy settings hide a certain field, it will be omitted from the results. + - The returned objects will be expunged, and changes to them will not be committed. + ''' + people = Person.search(criteria) + people_copy = [] + for person in people: + person_copy = copy(person) + make_transient(person_copy) + + persistent_data = person.get_persistent_data() + if persistent_data is not None: + for field in PERSISTENT_FIELDS: + setattr(person_copy, field, getattr(persistent_data, field)) + + people_copy.append(person_copy) + return people_copy class PersonPersistent(db.Model): __tablename__ = 'person_persistent' id = db.Column(db.Integer, primary_key=True, autoincrement=True) diff --git a/app/routes.py b/app/routes.py index a6443bc..2dede1c 100644 --- a/app/routes.py +++ b/app/routes.py @@ -1,7 +1,7 @@ from flask import render_template, make_response, request, redirect, url_for, jsonify, abort, g, session from app import app, db, scraper, cas from app.models import User, Person, Key, PersonPersistent -from app.util import requires_login, forbidden_via_api, to_json, get_now, succ, fail +from app.util import requires_login, forbidden_via_api, to_json, get_now, succ, fail, PERSISTENT_FIELDS from .cas_validate import validate from sqlalchemy import distinct @@ -299,17 +299,7 @@ def edit_post(): person_persistent = PersonPersistent(person_id=g.person.id) db.session.add(person_persistent) - for key in [ - 'socials_instagram', - 'socials_snapchat', - 'privacy_hide_image', - 'privacy_hide_email', - 'privacy_hide_room', - 'privacy_hide_phone', - 'privacy_hide_address', - 'privacy_hide_major', - 'privacy_hide_birthday' - ]: + for key in PERSISTENT_FIELDS: if key in payload: setattr(person_persistent, key, payload[key]) diff --git a/app/util.py b/app/util.py index 91ad4c1..bbc5358 100644 --- a/app/util.py +++ b/app/util.py @@ -71,3 +71,15 @@ def to_json(model): def get_now(): return int(datetime.datetime.utcnow().timestamp()) + +PERSISTENT_FIELDS = [ + 'socials_instagram', + 'socials_snapchat', + 'privacy_hide_image', + 'privacy_hide_email', + 'privacy_hide_room', + 'privacy_hide_phone', + 'privacy_hide_address', + 'privacy_hide_major', + 'privacy_hide_birthday' +] From e0f591aa2998c7aad68aba50cd97e8838e557e54 Mon Sep 17 00:00:00 2001 From: Eric Yoon Date: Mon, 18 Mar 2024 17:42:55 -0700 Subject: [PATCH 7/9] Privacy hiding working --- app/api.py | 7 ++++--- app/models.py | 4 +++- app/static/js/index.js | 2 ++ app/templates/base.html | 1 + app/util.py | 30 ++++++++++++++++++++++++++++++ 5 files changed, 40 insertions(+), 4 deletions(-) diff --git a/app/api.py b/app/api.py index 7f03fce..4c68848 100644 --- a/app/api.py +++ b/app/api.py @@ -50,7 +50,8 @@ def api_students(): if not criteria['filters'].get('school_code'): criteria['filters']['school_code'] = [] criteria['filters']['school_code'].append('YC') - students = Person.search(criteria) # TODO: Change this to respect privacy + students = Person.search_respect_privacy_include_persistent(criteria) + return to_json(students) @@ -61,8 +62,8 @@ def api_people(): criteria = request.get_json(force=True) or {} except: criteria = {} - people = Person.search(criteria) - # people = Person.search_respect_privacy_include_persistent(criteria) + people = Person.search_respect_privacy_include_persistent(criteria) + return to_json(people) diff --git a/app/models.py b/app/models.py index 958bc5c..76fbbb5 100644 --- a/app/models.py +++ b/app/models.py @@ -1,6 +1,6 @@ from app import app, db from app.search import SearchableMixin -from app.util import get_now, PERSISTENT_FIELDS +from app.util import get_now, PERSISTENT_FIELDS, scrub_hidden_data import jwt import datetime from copy import copy @@ -223,8 +223,10 @@ def search_respect_privacy_include_persistent(criteria): for field in PERSISTENT_FIELDS: setattr(person_copy, field, getattr(persistent_data, field)) + scrub_hidden_data(person_copy) people_copy.append(person_copy) return people_copy + class PersonPersistent(db.Model): __tablename__ = 'person_persistent' id = db.Column(db.Integer, primary_key=True, autoincrement=True) diff --git a/app/static/js/index.js b/app/static/js/index.js index 66babc9..3b98777 100644 --- a/app/static/js/index.js +++ b/app/static/js/index.js @@ -505,6 +505,8 @@ function loadNextPage() { addPill(pills, "major", "Major", "book", person); addPill(pills, "birthday", "Birthday", "birthday-cake", person); addPill(pills, "address", "Address", "home", person); + addPill(pills, "socials_instagram", "Instagram", "instagram", person); + addPill(pills, "socials_snapchat", "Snapchat", "snapchat", person); // Append the pills container to the person container personContainer.appendChild(pills); diff --git a/app/templates/base.html b/app/templates/base.html index 6ff2d87..c75079b 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -40,6 +40,7 @@