bvplan/bvplan/src/worker/update_info.rs
2023-02-27 11:09:16 +01:00

803 lines
30 KiB
Rust

use anyhow::{Context, Result};
use askama::Template;
use celery::error::TaskError;
use celery::task::TaskResult;
use chrono::prelude::*;
use diesel::prelude::*;
use fancy_regex::Regex;
use lazy_static::lazy_static;
use r2d2_redis::redis;
use std::fs;
use std::io::Write;
use std::path::Path;
use std::thread;
use std::time::Duration;
use crate::cache;
use crate::config;
use crate::db;
use crate::templates;
async fn get_schoolyear(client: &untis::Client, db_conn: &mut db::Connection) -> Result<i32> {
Ok(db::schema::schoolyears::table
.filter(db::schema::schoolyears::untis_id.eq(client.current_schoolyear().await?.id))
.select(db::schema::schoolyears::id)
.first(db_conn)?)
}
async fn fetch_current_tenant(
client: &untis::Client,
db_conn: &mut PgConnection,
schoolyear_id: i32,
) -> Result<()> {
let tenant = client.current_tenant().await?;
if diesel::select(diesel::dsl::not(diesel::dsl::exists(
db::schema::tenants::table.filter(db::schema::tenants::untis_id.eq(tenant.id)),
)))
.get_result::<bool>(db_conn)?
{
diesel::update(db::schema::tenants::table)
.filter(db::schema::tenants::active)
.set(db::schema::tenants::active.eq(false))
.execute(db_conn)?;
diesel::insert_into(db::schema::tenants::table)
.values(db::models::NewTenant {
untis_id: tenant.id,
schoolyear_id,
name: &tenant.display_name,
active: true,
})
.execute(db_conn)?;
} else if diesel::select(diesel::dsl::exists(
db::schema::tenants::table
.filter(db::schema::tenants::untis_id.eq(tenant.id))
.filter(db::schema::tenants::active.eq(false)),
))
.get_result::<bool>(db_conn)?
{
diesel::update(db::schema::tenants::table)
.filter(db::schema::tenants::active)
.set((
db::schema::tenants::active.eq(false),
db::schema::tenants::updated_at.eq(diesel::dsl::now),
))
.execute(db_conn)?;
diesel::update(db::schema::tenants::table)
.filter(db::schema::tenants::untis_id.eq(tenant.id))
.set((
db::schema::tenants::active.eq(true),
db::schema::tenants::updated_at.eq(diesel::dsl::now),
))
.execute(db_conn)?;
}
Ok(())
}
async fn fetch_timegrid(
client: &untis::Client,
db_conn: &mut PgConnection,
schoolyear_id: i32,
) -> Result<()> {
let days = client.timegrid().await?;
diesel::update(db::schema::timegrids::table)
.filter(db::schema::timegrids::active)
.set(db::schema::timegrids::active.eq(false))
.execute(db_conn)?;
let timegrid_id = diesel::insert_into(db::schema::timegrids::table)
.values(db::models::NewTimegrid {
schoolyear_id,
active: true,
})
.returning(db::schema::timegrids::id)
.get_result::<i32>(db_conn)?;
for day in days {
let timegrid_day_id = diesel::insert_into(db::schema::timegrid_days::table)
.values(db::models::NewTimegridDay {
timegrid_id,
weekday: day.day.try_into()?,
})
.returning(db::schema::timegrid_days::id)
.get_result::<i32>(db_conn)?;
diesel::insert_into(db::schema::timegrid_time_unit::table)
.values(
day.time_units
.into_iter()
.map(|x| db::models::NewTimegridTimeUnit {
timegrid_day_id,
start_time: x.start_time,
end_time: x.end_time,
})
.collect::<Vec<_>>(),
)
.execute(db_conn)?;
}
Ok(())
}
async fn fetch_teachers(
client: &untis::Client,
db_conn: &mut PgConnection,
schoolyear_id: i32,
) -> Result<()> {
let existing_teachers = db::schema::teachers::table
.select(db::schema::teachers::untis_id)
.filter(db::schema::teachers::schoolyear_id.eq(schoolyear_id))
.load::<i32>(db_conn)?;
diesel::insert_into(db::schema::teachers::table)
.values(
&client
.teachers()
.await?
.iter()
.filter(|t| !existing_teachers.contains(&t.id))
.map(|t| db::models::NewTeacher {
untis_id: t.id,
schoolyear_id,
name: &t.name,
forename: if t.forename.is_empty() {
None
} else {
Some(&t.forename)
},
display_name: &t.display_name,
})
.collect::<Vec<db::models::NewTeacher>>(),
)
.execute(db_conn)?;
Ok(())
}
async fn fetch_classes(
client: &untis::Client,
db_conn: &mut PgConnection,
schoolyear_id: i32,
) -> Result<()> {
let existing_classes = db::schema::classes::table
.select(db::schema::classes::untis_id)
.filter(db::schema::classes::schoolyear_id.eq(schoolyear_id))
.load::<i32>(db_conn)?;
diesel::insert_into(db::schema::classes::table)
.values(
&client
.classes()
.await?
.iter()
.filter(|c| !existing_classes.contains(&c.id))
.map(|c| db::models::NewClass {
untis_id: c.id,
schoolyear_id,
name: &c.name,
long_name: &c.long_name,
active: c.active,
})
.collect::<Vec<db::models::NewClass>>(),
)
.execute(db_conn)?;
Ok(())
}
async fn fetch_subjects(
client: &untis::Client,
db_conn: &mut PgConnection,
schoolyear_id: i32,
) -> Result<()> {
let existing_classes = db::schema::subjects::table
.select(db::schema::subjects::untis_id)
.filter(db::schema::subjects::schoolyear_id.eq(schoolyear_id))
.load::<i32>(db_conn)?;
diesel::insert_into(db::schema::subjects::table)
.values(
&client
.subjects()
.await?
.iter()
.filter(|c| !existing_classes.contains(&c.id))
.map(|c| db::models::NewSubject {
untis_id: c.id,
schoolyear_id,
name: &c.name,
long_name: &c.long_name,
})
.collect::<Vec<db::models::NewSubject>>(),
)
.execute(db_conn)?;
Ok(())
}
async fn fetch_rooms(
client: &untis::Client,
db_conn: &mut PgConnection,
schoolyear_id: i32,
) -> Result<()> {
let existing_classes = db::schema::rooms::table
.select(db::schema::rooms::untis_id)
.filter(db::schema::rooms::schoolyear_id.eq(schoolyear_id))
.load::<i32>(db_conn)?;
diesel::insert_into(db::schema::rooms::table)
.values(
&client
.rooms()
.await?
.iter()
.filter(|c| !existing_classes.contains(&c.id))
.map(|c| db::models::NewRoom {
untis_id: c.id,
schoolyear_id,
name: &c.name,
long_name: &c.long_name,
})
.collect::<Vec<db::models::NewRoom>>(),
)
.execute(db_conn)?;
Ok(())
}
async fn fetch_departments(
client: &untis::Client,
db_conn: &mut PgConnection,
schoolyear_id: i32,
) -> Result<()> {
let existing_classes = db::schema::departments::table
.select(db::schema::departments::untis_id)
.filter(db::schema::departments::schoolyear_id.eq(schoolyear_id))
.load::<i32>(db_conn)?;
diesel::insert_into(db::schema::departments::table)
.values(
&client
.departments()
.await?
.iter()
.filter(|c| !existing_classes.contains(&c.id))
.map(|c| db::models::NewDepartment {
untis_id: c.id,
schoolyear_id,
name: &c.name,
long_name: &c.long_name,
})
.collect::<Vec<db::models::NewDepartment>>(),
)
.execute(db_conn)?;
Ok(())
}
async fn fetch_holidays(
client: &untis::Client,
db_conn: &mut PgConnection,
schoolyear_id: i32,
) -> Result<()> {
let existing_classes = db::schema::holidays::table
.select(db::schema::holidays::untis_id)
.filter(db::schema::holidays::schoolyear_id.eq(schoolyear_id))
.load::<i32>(db_conn)?;
diesel::insert_into(db::schema::holidays::table)
.values(
&client
.holidays()
.await?
.iter()
.filter(|c| !existing_classes.contains(&c.id))
.map(|c| db::models::NewHoliday {
untis_id: c.id,
schoolyear_id,
name: &c.name,
long_name: &c.long_name,
start_date: c.start_date,
end_date: c.end_date,
})
.collect::<Vec<db::models::NewHoliday>>(),
)
.execute(db_conn)?;
Ok(())
}
async fn fetch_substitutions(
client: &untis::Client,
db_conn: &mut PgConnection,
schoolyear_id: i32,
) -> Result<i32> {
lazy_static! {
static ref TITLE_SELECTOR: scraper::Selector =
scraper::Selector::parse(".mon_title").unwrap();
static ref WEEK_TYPE_REGEX: Regex = Regex::new(r"\bWoche\s+\K\S+").unwrap();
}
let (date, week_type) = {
let html = reqwest::Client::new()
.get(&config::CONFIG.untis_vplan_url)
.header(
reqwest::header::USER_AGENT,
&config::CONFIG.untis_client_name,
)
.header(reqwest::header::ACCEPT, "text/html")
.basic_auth(
&config::CONFIG.untis_vplan_username,
Some(&config::CONFIG.untis_vplan_password),
)
.send()
.await?
.text()
.await?;
let document = scraper::Html::parse_document(&html);
let title = document
.select(&TITLE_SELECTOR)
.next()
.context("No element in vplan html which is selectable class \"mon_title\"")?
.text()
.next()
.context("\"mon_title\" element is empty")?;
(
NaiveDate::parse_from_str(
title
.split_once(' ')
.context("Could not split title string by whitespace")?
.0,
"%d.%m.%Y",
)?,
WEEK_TYPE_REGEX
.captures(title)?
.context("No week type could be found")?
.get(0)
.context("No week type could be found")?
.as_str()
.parse::<db::models::WeekType>()?,
)
};
let substitution_query_id = diesel::insert_into(db::schema::substitution_queries::table)
.values(db::models::NewSubstitutionQuery {
schoolyear_id,
date,
week_type,
queried_at: Utc::now().naive_utc(),
})
.returning(db::schema::substitution_queries::id)
.get_result::<i32>(db_conn)?;
for substitution in client.substitutions(&date, &date, None).await? {
let substitution_id = diesel::insert_into(db::schema::substitutions::table)
.values(db::models::NewSubstitution {
substitution_query_id,
subst_type: substitution.subst_type.into(),
lesson_id: substitution.lesson_id,
start_time: substitution.start_time,
end_time: substitution.end_time,
text: substitution.text.as_deref(),
})
.returning(db::schema::substitutions::id)
.get_result::<i32>(db_conn)?;
diesel::insert_into(db::schema::substitution_classes::table)
.values(
&substitution
.classes
.iter()
.enumerate()
.map(|(i, c)| {
Ok(db::models::NewSubstitutionClass {
substitution_id,
position: i as i16,
class_id: db::schema::classes::table
.filter(db::schema::classes::untis_id.eq(c.id))
.select(db::schema::classes::id)
.get_result::<i32>(db_conn)?,
original_id: if let Some(original_id) = c.original_id {
Some(
db::schema::classes::table
.filter(db::schema::classes::untis_id.eq(original_id))
.select(db::schema::classes::id)
.get_result::<i32>(db_conn)?,
)
} else {
None
},
})
})
.collect::<Result<Vec<db::models::NewSubstitutionClass>>>()?,
)
.execute(db_conn)?;
diesel::insert_into(db::schema::substitution_teachers::table)
.values(
&substitution
.teachers
.iter()
.enumerate()
.map(|(i, t)| {
Ok(db::models::NewSubstitutionTeacher {
substitution_id,
position: i as i16,
teacher_id: if t.id == 0 {
None
} else {
Some(
db::schema::teachers::table
.filter(db::schema::teachers::untis_id.eq(t.id))
.select(db::schema::teachers::id)
.get_result::<i32>(db_conn)?,
)
},
original_id: if let Some(original_id) = t.original_id {
Some(
db::schema::teachers::table
.filter(db::schema::teachers::untis_id.eq(original_id))
.select(db::schema::teachers::id)
.get_result::<i32>(db_conn)?,
)
} else {
None
},
})
})
.collect::<Result<Vec<db::models::NewSubstitutionTeacher>>>()?,
)
.execute(db_conn)?;
diesel::insert_into(db::schema::substitution_subjects::table)
.values(
&substitution
.subjects
.iter()
.enumerate()
.map(|(i, s)| {
Ok(db::models::NewSubstitutionSubject {
substitution_id,
position: i as i16,
subject_id: db::schema::subjects::table
.filter(db::schema::subjects::untis_id.eq(s.id))
.select(db::schema::subjects::id)
.get_result::<i32>(db_conn)?,
original_id: if let Some(original_id) = s.original_id {
Some(
db::schema::subjects::table
.filter(db::schema::subjects::untis_id.eq(original_id))
.select(db::schema::subjects::id)
.get_result::<i32>(db_conn)?,
)
} else {
None
},
})
})
.collect::<Result<Vec<db::models::NewSubstitutionSubject>>>()?,
)
.execute(db_conn)?;
diesel::insert_into(db::schema::substitution_rooms::table)
.values(
&substitution
.rooms
.iter()
.enumerate()
.map(|(i, r)| {
Ok(db::models::NewSubstitutionRoom {
substitution_id,
position: i as i16,
room_id: if r.id == 0 {
None
} else {
Some(
db::schema::rooms::table
.filter(db::schema::rooms::untis_id.eq(r.id))
.select(db::schema::rooms::id)
.get_result::<i32>(db_conn)?,
)
},
original_id: if let Some(original_id) = r.original_id {
Some(
db::schema::rooms::table
.filter(db::schema::rooms::untis_id.eq(original_id))
.select(db::schema::rooms::id)
.get_result::<i32>(db_conn)?,
)
} else {
None
},
})
})
.collect::<Result<Vec<db::models::NewSubstitutionRoom>>>()?,
)
.execute(db_conn)?;
}
Ok(substitution_query_id)
}
type StartEndTime = (NaiveTime, NaiveTime);
fn get_period(times: &Vec<StartEndTime>, start: bool, time: NaiveTime) -> Option<usize> {
times
.iter()
.position(|x| (if start { x.0 } else { x.1 }) == time)
.map(|x| x + 1)
}
fn cache_substitutions(
db_conn: &mut PgConnection,
redis_conn: &mut cache::Connection,
substitution_query_id: i32,
last_import_time: NaiveDateTime,
) -> Result<()> {
let (queried_at, date, week_type) = db::schema::substitution_queries::table
.select((
db::schema::substitution_queries::queried_at,
db::schema::substitution_queries::date,
db::schema::substitution_queries::week_type,
))
.filter(db::schema::substitution_queries::id.eq(substitution_query_id))
.first::<(NaiveDateTime, NaiveDate, db::models::WeekType)>(db_conn)?;
let weekday: db::models::Weekday = date.weekday().into();
let timegrid_id = db::schema::timegrids::table
.select(db::schema::timegrids::id)
.filter(db::schema::timegrids::active)
.first::<i32>(db_conn)?;
let day_id: i32 = db::schema::timegrid_days::table
.select((
db::schema::timegrid_days::id,
db::schema::timegrid_days::weekday,
))
.filter(db::schema::timegrid_days::timegrid_id.eq(timegrid_id))
.load::<(i32, db::models::Weekday)>(db_conn)?
.into_iter()
.find(|x| x.1 == weekday)
.context("Could not find timegrid day")?
.0;
let times: Vec<StartEndTime> = db::schema::timegrid_time_unit::table
.select((
db::schema::timegrid_time_unit::start_time,
db::schema::timegrid_time_unit::end_time,
))
.filter(db::schema::timegrid_time_unit::timegrid_day_id.eq(day_id))
.order(db::schema::timegrid_time_unit::start_time.asc())
.load::<StartEndTime>(db_conn)?;
let query = cache::keys::substitutions::SubstitutionQuery {
date,
week_type,
queried_at,
last_import_time,
schoolyear: "schoolyear".to_string(),
tenant: "OHG Furtwangen".to_string(),
substitutions: db::schema::substitutions::table
.filter(db::schema::substitutions::substitution_query_id.eq(substitution_query_id))
.load::<db::models::Substitution>(db_conn)
.unwrap()
.into_iter()
.map(|s| {
if let Some(subst_subject) = db::schema::substitution_subjects::table
.filter(db::schema::substitution_subjects::substitution_id.eq(s.id))
.order(db::schema::substitution_subjects::position.asc())
.first::<db::models::SubstitutionSubject>(db_conn)
.optional()?
{
let subst_class_ids = db::schema::substitution_classes::table
.select(db::schema::substitution_classes::class_id)
.filter(db::schema::substitution_classes::substitution_id.eq(s.id))
.order(db::schema::substitution_classes::position.asc())
.load::<i32>(db_conn)?;
let classes = db::schema::classes::table
.select(db::schema::classes::name)
.filter(db::schema::classes::id.eq_any(subst_class_ids))
.load::<String>(db_conn)?;
let subst_teachers = db::schema::substitution_teachers::table
.filter(db::schema::substitution_teachers::substitution_id.eq(s.id))
.order(db::schema::substitution_teachers::position.asc())
.load::<db::models::SubstitutionTeacher>(db_conn)?;
let (prev_room, room) = if let Some(r) = db::schema::substitution_rooms::table
.filter(db::schema::substitution_rooms::substitution_id.eq(s.id))
.order(db::schema::substitution_rooms::position.asc())
.first::<db::models::SubstitutionRoom>(db_conn)
.optional()?
{
let name = if let Some(id) = r.room_id {
Some(
db::schema::rooms::table
.select(db::schema::rooms::name)
.filter(db::schema::rooms::id.eq(id))
.first::<String>(db_conn)?,
)
} else {
None
};
match s.subst_type {
db::models::SubstitutionType::Cancel => (name, None),
_ => (
if let Some(id) = r.original_id {
Some(
db::schema::rooms::table
.select(db::schema::rooms::name)
.filter(db::schema::rooms::id.eq(id))
.first::<String>(db_conn)?,
)
} else {
None
},
name,
),
}
} else {
(None, None)
};
let prev_subject = db::schema::subjects::table
.select(db::schema::subjects::name)
.filter(db::schema::subjects::id.eq(subst_subject.subject_id))
.first::<String>(db_conn)?;
let start_period = get_period(&times, true, s.start_time)
.context("Could not find period from start time")?;
Ok(Some(cache::keys::substitutions::Substitution {
time: (start_period, None),
classes,
prev_subject: prev_subject.to_owned(),
subject: match s.subst_type {
db::models::SubstitutionType::Cancel => None,
_ => match subst_subject.original_id {
Some(id) => Some(
db::schema::subjects::table
.select(db::schema::subjects::name)
.filter(db::schema::subjects::id.eq(id))
.first::<String>(db_conn)?,
),
None => Some(prev_subject),
},
},
teachers: match s.subst_type {
db::models::SubstitutionType::Cancel => vec![],
_ => db::schema::teachers::table
.select(db::schema::teachers::display_name)
.filter(
db::schema::teachers::id.eq_any(
subst_teachers
.iter()
.filter_map(|t| t.teacher_id)
.collect::<Vec<i32>>(),
),
)
.load::<String>(db_conn)?,
},
prev_room,
room,
text: s.text,
}))
} else {
Ok(None)
}
})
.collect::<Result<Vec<_>>>()
.unwrap()
.into_iter()
.filter_map(|s| s)
.collect::<Vec<_>>(),
};
fs::write(
Path::new("/backups").join(format!(
"{}_{}_{}.json.gz",
date, queried_at, substitution_query_id
)),
{
let mut e = flate2::write::ZlibEncoder::new(vec![], flate2::Compression::best());
e.write_all(serde_json::to_string(&query)?.as_bytes())?;
e.finish()?
},
)?;
redis::cmd("SET")
.arg(cache::keys::SUBSTITUTIONS_HTML)
.arg(minify_html::minify(
templates::BVplan { data: query }.render()?.as_bytes(),
&minify_html::Cfg {
do_not_minify_doctype: false,
ensure_spec_compliant_unquoted_attribute_values: false,
keep_spaces_between_attributes: false,
minify_css: true,
minify_js: true,
..Default::default()
},
))
.query::<()>(redis_conn)?;
Ok(())
}
#[celery::task]
pub async fn update_info() -> TaskResult<()> {
let dur = Duration::from_secs(2);
thread::sleep(dur);
let mut client = match config::untis_from_env() {
Ok(x) => x,
Err(e) => return Err(TaskError::UnexpectedError(format!("{:?}", e))),
};
if let Err(e) = client.login().await {
return Err(TaskError::UnexpectedError(format!("{:?}", e)));
}
thread::sleep(dur);
let db_conn = &mut match db::POOL.get() {
Ok(x) => x,
Err(e) => return Err(TaskError::UnexpectedError(format!("{:?}", e))),
};
let redis_conn = &mut match cache::POOL.get() {
Ok(x) => x,
Err(e) => return Err(TaskError::UnexpectedError(format!("{:?}", e))),
};
let schoolyear_id = match get_schoolyear(&client, db_conn).await {
Ok(x) => x,
Err(e) => return Err(TaskError::UnexpectedError(format!("{:?}", e))),
};
thread::sleep(dur);
if let Err(e) = fetch_current_tenant(&client, db_conn, schoolyear_id).await {
return Err(TaskError::UnexpectedError(format!("{:?}", e)));
}
thread::sleep(dur);
if let Err(e) = fetch_timegrid(&client, db_conn, schoolyear_id).await {
return Err(TaskError::UnexpectedError(format!("{:?}", e)));
}
thread::sleep(dur);
if let Err(e) = fetch_teachers(&client, db_conn, schoolyear_id).await {
return Err(TaskError::UnexpectedError(format!("{:?}", e)));
}
thread::sleep(dur);
if let Err(e) = fetch_classes(&client, db_conn, schoolyear_id).await {
return Err(TaskError::UnexpectedError(format!("{:?}", e)));
}
thread::sleep(dur);
if let Err(e) = fetch_subjects(&client, db_conn, schoolyear_id).await {
return Err(TaskError::UnexpectedError(format!("{:?}", e)));
}
thread::sleep(dur);
if let Err(e) = fetch_rooms(&client, db_conn, schoolyear_id).await {
return Err(TaskError::UnexpectedError(format!("{:?}", e)));
}
thread::sleep(dur);
if let Err(e) = fetch_departments(&client, db_conn, schoolyear_id).await {
return Err(TaskError::UnexpectedError(format!("{:?}", e)));
}
thread::sleep(dur);
if let Err(e) = fetch_holidays(&client, db_conn, schoolyear_id).await {
return Err(TaskError::UnexpectedError(format!("{:?}", e)));
}
thread::sleep(dur);
let last_import_time = match client.last_import_time().await {
Ok(x) => x,
Err(e) => return Err(TaskError::UnexpectedError(format!("{:?}", e))),
};
thread::sleep(dur);
let substitution_query_id = match fetch_substitutions(&client, db_conn, schoolyear_id).await {
Ok(x) => x,
Err(e) => return Err(TaskError::UnexpectedError(format!("{:?}", e))),
};
thread::sleep(dur);
if let Err(e) = client.logout().await {
return Err(TaskError::UnexpectedError(format!("{:?}", e)));
}
// thread::sleep(dur);
if let Err(e) =
cache_substitutions(db_conn, redis_conn, substitution_query_id, last_import_time)
{
return Err(TaskError::UnexpectedError(format!("{:?}", e)));
}
Ok(())
}