From bd8f27cab3927c63984a6ec02378c2d59f4b8d2f Mon Sep 17 00:00:00 2001 From: Daniel Bulant Date: Sat, 23 Oct 2021 22:38:06 +0200 Subject: [PATCH] mysql tables and import --- utils/api.js | 70 +++++++++++++++++++- utils/db/import.js | 111 +++++++++++++++++++++++++++----- utils/models/class.js | 4 +- utils/models/event.js | 4 +- utils/models/event_group.js | 7 +- utils/models/event_student.js | 7 +- utils/models/group.js | 12 +--- utils/models/group_student.js | 4 +- utils/models/person.js | 41 ++++++++++++ utils/models/rickroll.js | 25 +------ utils/models/room.js | 4 +- utils/models/student.js | 25 ++----- utils/models/subject.js | 2 +- utils/models/supplementation.js | 42 +----------- utils/models/teacher.js | 2 +- utils/models/timetable.js | 45 ------------- utils/sequelize.js | 62 +++++++++++++----- 17 files changed, 279 insertions(+), 188 deletions(-) create mode 100644 utils/models/person.js diff --git a/utils/api.js b/utils/api.js index d529a2f..cd27eaf 100644 --- a/utils/api.js +++ b/utils/api.js @@ -2,7 +2,7 @@ const { DateTime } = require("luxon"); const fetch = require("node-fetch"); async function request(endpoint, body) { - const res = await fetch("https://www.ssps.cz/" + endpoint, { + const res = await fetch(endpoint.startsWith("https://") ? endpoint : "https://www.ssps.cz/" + endpoint, { method: body ? "POST" : "GET", body, headers: { @@ -132,8 +132,61 @@ class Schedule { * @property {{ href: string, count: number }[]} version-history */ +/** + * @typedef ProfesniUzivatel + * @property {string|null} about + * @property {string|null} birthday + * @property {boolean} blocked + * @property {string|null} class + * @property {number} documentId + * @property {string} email + * @property {string} firstName + * @property {string} lastName + * @property {number} id + * @property {string} imagePath + * @property {number} similarity + */ + +class ProfesniSitAPI { + async *getUsers() { + var errors = 0; + var id = 1; + while(errors < 15) { + /** @type {ProfesniUzivatel|null} */ + const res = await request(`https://profesnisit.ssps.cz/user/get-user?id=${id}`); + id++; + console.log("Trying", id); + if(!res || Object.values(res).every(t => t === null)) { + errors++; + continue; + } + errors = 0; + yield res; + } + } +} + +const letterMap = { + "ě": "e", + "š": "s", + "č": "c", + "ř": "r", + "ž": "z", + "ý": "y", + "á": "a", + "í": "i", + "é": "e", + "ó": "o", + "ú": "u", + "ů": "u" +}; +function removeCestina(str) { + return str.split("").map(t => letterMap[t] || t).join(""); +} + class API { request = request; + profesniSit = new ProfesniSitAPI; async getSupplementations(date = new DateTime) { const res = await request(`wp-content/themes/ssps-wordpress-theme/supplementation.php/?date=${date.toFormat("yyyyMMdd")}`); @@ -162,6 +215,21 @@ class API { return room.toString().padStart(3, "0"); } + isTeacherMail(mail) { + return /[a-z.]+@ssps\.cz/.test(mail); + } + isStudentMail(mail) { + return /[a-z]+\.[a-z]{2}\.20[12][0-9]@ssps\.cz/.test(mail); + } + + buildTeacherMail(name) { + return `${removeCestina(name).replace(/ /g, ".")}@ssps.cz`.toLowerCase(); + } + buildStudentMail(name, year) { + if(!Array.isArray(name)) name = name.split(" "); + return `${removeCestina(name[1])}.${removeCestina(name[0]).substr(0,2)}.${year}@ssps.cz`.toLowerCase(); + } + groups = [ ["celá třída"], ["Skupina 1", "Skupina 2"], diff --git a/utils/db/import.js b/utils/db/import.js index 6602c5c..a594afa 100644 --- a/utils/db/import.js +++ b/utils/db/import.js @@ -15,11 +15,31 @@ const Subject = require("../models/subject"); const Teacher = require("../models/teacher"); const Room = require("../models/room"); const Timetable = require("../models/timetable"); +const Person = require("../models/person"); +const Student = require("../models/student"); +const { Op } = require("sequelize"); + +function catcher(err) { + if(err.original && err.original.text.startsWith("Duplicate entry")) return null; + throw err; +} + +function contextErrors(data) { + return (err) => { + try { + catcher(err); + } catch(e) { + console.warn(data); + throw e; + } + } +} sequelize.afterBulkSync(async () => { + if(!isMain) return; console.log("Sync in progress"); - if(isMain) { + if(process.argv.includes("--timetable")) { console.log("Syncing timetable"); const classes = Object.values(api.map); await Promise.all(classes.map(async id => { @@ -29,7 +49,7 @@ sequelize.afterBulkSync(async () => { year: (2021 - parseInt(name[0]-1)), type: name[1], discord: server.reverseRoles[name] - }).catch(console.error); + }).catch(contextErrors(name)); const timetable = await api.getSchedule(id); for(const day in timetable.schedule) { for(let hour in timetable.schedule[day]) { @@ -40,35 +60,92 @@ sequelize.afterBulkSync(async () => { id: item.Group.Id, abbrev: item.Group.Abbrev, name: item.Group.Name, - class: id - }).catch(console.error); + classId: id + }).catch(contextErrors(item)); await Subject.create({ id: item.Subject.Id, - abrev: item.Subject.Abbrev, + abbrev: item.Subject.Abbrev, name: item.Subject.Name - }).catch(console.error); + }).catch(contextErrors(item)); + await Person.create({ + name: item.Teacher.Name, + mail: api.buildTeacherMail(item.Teacher.Name), + flags: 1 + }).catch(contextErrors(item)); + const person = await Person.findOne({ where: { mail: api.buildTeacherMail(item.Teacher.Name) } }); await Teacher.create({ id: item.Teacher.Id, abbrev: item.Teacher.Abbrev, - name: item.Teacher.Name - }); - await Room.create({ id: item.Room.Abbrev }); - const cycle = item.Cycles[0].Id === "1" && item.Cycles[1].Id === "2" ? "always" : item.Cycles[0].Id === "1" ? "even" : item.Cycles[0].Id === "2" ? "odd" : null; + name: item.Teacher.Name, + personId: person.id + }).catch(contextErrors(item)); + if(item.Room.Abbrev) await Room.create({ id: item.Room.Abbrev }).catch(contextErrors(item)); + const cycle = item.Cycles.length > 1 && item.Cycles[0].Id === "1" && item.Cycles[1].Id === "2" ? "always" : item.Cycles[0].Id === "1" ? "even" : item.Cycles[0].Id === "2" ? "odd" : null; await Timetable.create({ day, hour, - class: id, - group: item.Group.Id, - subject: item.Subject.Id, - teacher: item.Teacher.Id, - room: item.Room.Abbrev, + classId: id, + groupId: item.Group.Id, + subjectId: item.Subject.Id, + teacherId: item.Teacher.Id, + roomId: item.Room.Abbrev || null, cycles: cycle - }); + }).catch(contextErrors(item)); console.log("Done", day, hour, item.Subject.Id); } } } })); - console.log("Sync done"); + console.log("Timetables synced") } + if(process.argv.includes("--profesni-sit")) { + /** + * V tuhle chvíly je poslední existující ID 414 (@2021-10-23) + * get-users vrací jen prvních 50. Snažím se zjistit jak se dostat dál... + */ + console.log("Syncing profesni sit"); + for await(const user of api.profesniSit.getUsers()) { + if(!user.email) continue; + if(!user.email.endsWith("ssps.cz")) continue; + if(user.email.endsWith("@skola.ssps.cz")) user.email = user.email.replace("@skola.ssps.cz", "@ssps.cz"); + console.log(user.email); + var person = await Person.findOne({ + where: { + [Op.or]: { + mail: user.email, + name: `${user.firstName} ${user.lastName}` + } + } + }); + if(!person) person = Person.build({ mail: user.email, flags: 1 }); + if(user.about) person.about = user.about; + if(user.birthday) person.birthday = new Date(user.birthday); + person.name = `${user.firstName} ${user.lastName}`; + if(user.imagePath) person.avatar = "https://profesnisit.ssps.cz/" + user.imagePath; + await person.save(); + + if(api.isStudentMail(user.email)) { + if(!user.class) continue; + user.class = user.class.replace(/ |\./, "").toUpperCase(); + var student = await Student.findOne({ + where: { + personId: person.id + } + }); + if(!student) { + await Person.create({ + personId: person.id, + name: person.name + }); + } else { + student.name = person.name; + await student.save(); + } + } else if(api.isTeacherMail(user.email)) { + // synced by timetables + } + } + console.log("Profesni sit synced"); + } + console.log("Sync done"); }); \ No newline at end of file diff --git a/utils/models/class.js b/utils/models/class.js index 9420a35..27bb6c4 100644 --- a/utils/models/class.js +++ b/utils/models/class.js @@ -2,7 +2,7 @@ const { Sequelize, Op, Model, DataTypes } = require("sequelize"); const sequelize = require("../sequelize"); const Class = sequelize.define( - "classes", + "class", { id: { type: DataTypes.CHAR(2), @@ -22,7 +22,7 @@ const Class = sequelize.define( type: DataTypes.STRING }, displayName: { - type: "CHAR(2) GENERATED ALWAYS AS concat(`year` - year(curdate()) + 1,`type`) STORED", + type: "CHAR(2) GENERATED ALWAYS AS (concat(year(curdate()) - `year` + 1, `type`))", set() { throw new Error('displayName is read-only') } diff --git a/utils/models/event.js b/utils/models/event.js index c420fe9..eb2c733 100644 --- a/utils/models/event.js +++ b/utils/models/event.js @@ -2,7 +2,7 @@ const { Sequelize, Op, Model, DataTypes } = require("sequelize"); const sequelize = require("../sequelize"); const Event = sequelize.define( - "events", + "event", { id: { type: DataTypes.INTEGER, @@ -11,7 +11,7 @@ const Event = sequelize.define( }, date: { type: DataTypes.DATE, - defaultValue: "CURRENT_TIMESTAMP()", + defaultValue: Sequelize.fn("CURRENT_TIMESTAMP"), allowNull: false }, type: { diff --git a/utils/models/event_group.js b/utils/models/event_group.js index 225955a..82907f3 100644 --- a/utils/models/event_group.js +++ b/utils/models/event_group.js @@ -4,7 +4,7 @@ const Event = require("./event"); const Group = require("./group"); const EventGroup = sequelize.define( - "event_groups", + "event_group", { event: { type: DataTypes.INTEGER, @@ -21,6 +21,9 @@ const EventGroup = sequelize.define( model: Group, key: "id" } + }, + relation: { + type: DataTypes.STRING } }, { @@ -30,7 +33,5 @@ const EventGroup = sequelize.define( }] } ); -EventGroup.belongsTo(Event, { as: "event" }); -EventGroup.belongsTo(Group, { as: "group" }); module.exports = EventGroup; \ No newline at end of file diff --git a/utils/models/event_student.js b/utils/models/event_student.js index 888c3ca..47a63a8 100644 --- a/utils/models/event_student.js +++ b/utils/models/event_student.js @@ -4,7 +4,7 @@ const Event = require("./event"); const Student = require("./student"); const EventStudent = sequelize.define( - "event_students", + "event_student", { event: { type: DataTypes.INTEGER, @@ -21,6 +21,9 @@ const EventStudent = sequelize.define( model: Student, key: "id" } + }, + relation: { + type: DataTypes.STRING } }, { @@ -30,7 +33,5 @@ const EventStudent = sequelize.define( }] } ); -EventStudent.belongsTo(Event, { as: "event" }); -EventStudent.belongsTo(Student, { as: "student" }); module.exports = EventStudent; \ No newline at end of file diff --git a/utils/models/group.js b/utils/models/group.js index 726a372..ba76ce0 100644 --- a/utils/models/group.js +++ b/utils/models/group.js @@ -1,9 +1,8 @@ const { Sequelize, Op, Model, DataTypes } = require("sequelize"); const sequelize = require("../sequelize"); -const Class = require("./class"); const Group = sequelize.define( - "groups", + "group", { id: { type: DataTypes.STRING(16), @@ -15,15 +14,10 @@ const Group = sequelize.define( name: { type: DataTypes.STRING }, - class: { - type: DataTypes.CHAR(2), - references: { - model: Class, - key: "id" - } + discord: { + type: DataTypes.STRING } } ); -Group.belongsTo(Class, { as: "class" }); module.exports = Group; \ No newline at end of file diff --git a/utils/models/group_student.js b/utils/models/group_student.js index a5ba8f0..aa89620 100644 --- a/utils/models/group_student.js +++ b/utils/models/group_student.js @@ -4,7 +4,7 @@ const Group = require("./group"); const Student = require("./student"); const GroupStudent = sequelize.define( - "group_students", + "group_student", { group: { type: DataTypes.STRING(16), @@ -30,7 +30,5 @@ const GroupStudent = sequelize.define( }] } ); -EventStudent.belongsTo(Student, { as: "student" }); -EventStudent.belongsTo(Group, { as: "group" }); module.exports = GroupStudent; \ No newline at end of file diff --git a/utils/models/person.js b/utils/models/person.js new file mode 100644 index 0000000..86feee2 --- /dev/null +++ b/utils/models/person.js @@ -0,0 +1,41 @@ +const { Sequelize, Op, Model, DataTypes } = require("sequelize"); +const sequelize = require("../sequelize"); + +const Person = sequelize.define( + "person", + { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true + }, + name: { + type: DataTypes.STRING + }, + mail: { + type: DataTypes.STRING + }, + avatar: { + type: DataTypes.STRING + }, + about: { + type: DataTypes.STRING(512) + }, + birthday: { + type: DataTypes.DATEONLY + }, + discord: { + type: DataTypes.BIGINT + }, + flags: { + type: DataTypes.INTEGER + } + }, { + indexes: [{ + fields: ["mail"], + type: "UNIQUE" + }] + } +); + +module.exports = Person; \ No newline at end of file diff --git a/utils/models/rickroll.js b/utils/models/rickroll.js index af4960f..975e733 100644 --- a/utils/models/rickroll.js +++ b/utils/models/rickroll.js @@ -1,42 +1,19 @@ const { Sequelize, Op, Model, DataTypes } = require("sequelize"); const sequelize = require("../sequelize"); -const Student = require("./student"); const Rickroll = sequelize.define( - "rickrolls", + "rickroll", { id: { type: DataTypes.INTEGER, autoIncrement: true, primaryKey: true }, - source: { - type: DataTypes.STRING(45), - allowNull: false, - references: { - key: "id", - model: Student - } - }, - target: { - type: DataTypes.STRING(45), - allowNull: false, - references: { - key: "id", - model: Student - } - }, - date: { - type: DataTypes.DATE, - defaultValue: "CURRENT_TIMESTAMP()" - }, link: { type: DataTypes.STRING(255), defaultValue: "'https://www.youtube.com/watch?v=dQw4w9WgXcQ'" } } ); -EventGroup.belongsTo(Student, { as: "source" }); -EventGroup.belongsTo(Student, { as: "target" }); module.exports = Rickroll; \ No newline at end of file diff --git a/utils/models/room.js b/utils/models/room.js index 4a6fdad..efeb473 100644 --- a/utils/models/room.js +++ b/utils/models/room.js @@ -2,10 +2,10 @@ const { Sequelize, Op, Model, DataTypes } = require("sequelize"); const sequelize = require("../sequelize"); const Room = sequelize.define( - "rooms", + "room", { id: { - type: DataTypes.INTEGER, + type: DataTypes.STRING(8), primaryKey: true } } diff --git a/utils/models/student.js b/utils/models/student.js index 3332f0e..44984bb 100644 --- a/utils/models/student.js +++ b/utils/models/student.js @@ -1,36 +1,23 @@ const { Sequelize, Op, Model, DataTypes } = require("sequelize"); const sequelize = require("../sequelize"); -const Class = require("./class"); const Student = sequelize.define( - "students", + "student", { id: { type: DataTypes.STRING(45), primaryKey: true, allowNull: false }, - class: { - type: DataTypes.CHAR(2), - allowNull: false, - references: { - model: Class, - key: "id" - } - }, name: { type: DataTypes.STRING(45), allowNull: false - }, - born: { - type: DataTypes.DATEONLY - }, - discord: { - type: DataTypes.BIGINT - }, - flags: { - type: DataTypes.INTEGER } + }, { + indexes: [{ + fields: ["personId"], + type: "UNIQUE" + }] } ); diff --git a/utils/models/subject.js b/utils/models/subject.js index 656961b..49b9bd1 100644 --- a/utils/models/subject.js +++ b/utils/models/subject.js @@ -2,7 +2,7 @@ const { Sequelize, Op, Model, DataTypes } = require("sequelize"); const sequelize = require("../sequelize"); const Subject = sequelize.define( - "subjects", + "subject", { id: { type: DataTypes.CHAR(3), diff --git a/utils/models/supplementation.js b/utils/models/supplementation.js index 5e62766..35ebbaa 100644 --- a/utils/models/supplementation.js +++ b/utils/models/supplementation.js @@ -1,13 +1,8 @@ const { Sequelize, Op, Model, DataTypes } = require("sequelize"); const sequelize = require("../sequelize"); -const Class = require("./class"); -const Group = require("./group"); -const Room = require("./room"); -const Subject = require("./subject"); -const Teacher = require("./teacher"); const Supplementation = sequelize.define( - "supplementations", + "supplementation", { id: { type: DataTypes.INTEGER, @@ -18,13 +13,6 @@ const Supplementation = sequelize.define( type: DataTypes.DATEONLY, allowNull: false }, - class: { - type: DataTypes.CHAR(2), - references: { - model: Class, - key: "id" - } - }, type: { type: DataTypes.ENUM, values: ["substitutes", "move-src", "move-target", "join"] @@ -32,34 +20,6 @@ const Supplementation = sequelize.define( hour: { type: DataTypes.TINYINT }, - subject: { - type: DataTypes.CHAR(3), - references: { - model: Subject, - key: "id" - } - }, - group: { - type: DataTypes.STRING(16), - references: { - model: Group, - key: "id" - } - }, - room: { - type: DataTypes.INTEGER, - references: { - model: Room, - key: "id" - } - }, - teacher: { - type: DataTypes.STRING(16), - references: { - model: Teacher, - key: "abbrev" - } - }, notes: { type: DataTypes.STRING(255) } diff --git a/utils/models/teacher.js b/utils/models/teacher.js index 356c151..96bbdc3 100644 --- a/utils/models/teacher.js +++ b/utils/models/teacher.js @@ -2,7 +2,7 @@ const { Sequelize, Op, Model, DataTypes } = require("sequelize"); const sequelize = require("../sequelize"); const Teacher = sequelize.define( - "teachers", + "teacher", { id: { type: DataTypes.STRING(16), diff --git a/utils/models/timetable.js b/utils/models/timetable.js index 78a31bd..8b403a7 100644 --- a/utils/models/timetable.js +++ b/utils/models/timetable.js @@ -1,10 +1,5 @@ const { Sequelize, Op, Model, DataTypes } = require("sequelize"); const sequelize = require("../sequelize"); -const Class = require("./class"); -const Group = require("./group"); -const Room = require("./room"); -const Subject = require("./subject"); -const Teacher = require("./teacher"); const Timetable = sequelize.define( "timetable", @@ -22,46 +17,6 @@ const Timetable = sequelize.define( type: DataTypes.TINYINT, allowNull: false }, - class: { - type: DataTypes.CHAR(2), - references: { - model: Class, - key: "id" - }, - allowNull: false - }, - group: { - type: DataTypes.STRING(16), - references: { - model: Group, - key: "id" - }, - allowNull: false - }, - subject: { - type: DataTypes.CHAR(3), - references: { - model: Subject, - key: "id" - }, - allowNull: false - }, - teacher: { - type: DataTypes.STRING(16), - references: { - model: Teacher, - key: "id" - }, - allowNull: false - }, - room: { - type: DataTypes.INTEGER, - references: { - models: Room, - key: "id" - }, - allowNull: false - }, cycles: { type: DataTypes.ENUM, values: ["always", "odd", "even"] diff --git a/utils/sequelize.js b/utils/sequelize.js index 7ca7a29..8606e45 100644 --- a/utils/sequelize.js +++ b/utils/sequelize.js @@ -1,22 +1,54 @@ const { Sequelize } = require("sequelize"); -const sequelize = new Sequelize("prestiz", config.user, config.password, { - host: config.host, +const sequelize = new Sequelize("prestiz", global.config.mysql.user, global.config.mysql.password, { + host: global.config.mysql.host, dialect: "mariadb", - define: { - freezeTableName: true - } + logging: false }); module.exports = sequelize; -require("./models/class"); -require("./models/student"); -require("./models/room"); -require("./models/group"); -require("./models/teacher"); -require("./models/subject"); -require("./models/timetable"); -require("./models/rickroll"); -require("./models/supplementation"); +const Class = require("./models/class"); +const Person = require("./models/person"); +const Student = require("./models/student"); +const Room = require("./models/room"); +const Group = require("./models/group"); +const Teacher = require("./models/teacher"); +const Subject = require("./models/subject"); +const Timetable = require("./models/timetable"); +const Rickroll = require("./models/rickroll"); +const Supplementation = require("./models/supplementation"); +const Event = require("./models/event"); +const EventStudent = require("./models/event_student"); +const EventGroup = require("./models/event_group"); +const GroupStudent = require("./models/group_student"); -sequelize.sync(); \ No newline at end of file +Supplementation.belongsTo(Class); +Supplementation.belongsTo(Subject); +Supplementation.belongsTo(Group); +Supplementation.belongsTo(Room); +Supplementation.belongsTo(Teacher); + +Rickroll.belongsTo(Person, { as: "source" }); +Rickroll.belongsTo(Person, { as: "target" }); + +Student.belongsTo(Class); +Group.belongsTo(Class); + +Student.belongsToMany(Event, { through: EventStudent }); +Event.belongsToMany(Student, { through: EventStudent }); +Group.belongsToMany(Event, { through: EventGroup }); +Event.belongsToMany(Group, { through: EventGroup }); + +Group.belongsToMany(Student, { through: GroupStudent }); +Student.belongsToMany(Group, { through: GroupStudent }); + +Timetable.belongsTo(Class); +Timetable.belongsTo(Group); +Timetable.belongsTo(Subject); +Timetable.belongsTo(Teacher); +Timetable.belongsTo(Room); + +Person.hasOne(Student); +Person.hasOne(Teacher); + +sequelize.sync({ force: global.config.mysql.forceUpdate }); \ No newline at end of file