dergrimm/bvplan
dergrimm/bvplan
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Update

Browse source
This commit is contained in:
Dominic Grimm 2023-01-17 06:54:45 +01:00
parent f614e606f4
commit a177f2a5d4
No known key found for this signature in database
GPG key ID: 6F294212DEAAC530
22 changed files with 1339 additions and 667 deletions
Download patch file Download diff file Expand all files Collapse all files

406
backend/src/worker/update_info.rs
View file

@ -1,17 +1,20 @@
use anyhow::Result;
use anyhow::{bail, Context, Result};
use celery::error::TaskError;
use celery::task::TaskResult;
use chrono::prelude::*;
use diesel::prelude::*;
use lazy_static::lazy_static;
use r2d2_redis::redis;
use std::ops::DerefMut;
use std::thread;
use std::time::Duration;
use crate::{config, db};
use crate::{cache, config, db};
async fn fetch_schoolyears(client: &untis::Client, conn: &mut PgConnection) -> Result<i32> {
async fn fetch_schoolyears(client: &untis::Client, db_conn: &mut PgConnection) -> Result<i32> {
let existing_schoolyears = db::schema::schoolyears::table
.select(db::schema::schoolyears::untis_id)
.load::<i32>(conn)?;
.load::<i32>(db_conn)?;
diesel::insert_into(db::schema::schoolyears::table)
.values(
&client
@ -27,29 +30,29 @@ async fn fetch_schoolyears(client: &untis::Client, conn: &mut PgConnection) -> R
})
.collect::<Vec<db::models::NewSchoolyear>>(),
)
.execute(conn)?;
.execute(db_conn)?;
Ok(db::schema::schoolyears::table
.filter(db::schema::schoolyears::untis_id.eq(client.current_schoolyear().await?.id))
.select(db::schema::schoolyears::id)
.first(conn)?)
.first(db_conn)?)
}
async fn fetch_current_tenant(
client: &untis::Client,
conn: &mut PgConnection,
db_conn: &mut PgConnection,
schoolyear_id: i32,
) -> Result<()> {
let tenant = client.current_tenant().await?;
if diesel::select(diesel::dsl::not(diesel::dsl::exists(
db::schema::tenants::table.filter(db::schema::tenants::untis_id.eq(tenant.id)),
)))
.get_result::<bool>(conn)?
.get_result::<bool>(db_conn)?
{
diesel::update(db::schema::tenants::table)
.filter(db::schema::tenants::active)
.set(db::schema::tenants::active.eq(false))
.execute(conn)?;
.execute(db_conn)?;
diesel::insert_into(db::schema::tenants::table)
.values(db::models::NewTenant {
untis_id: tenant.id,
@ -57,13 +60,13 @@ async fn fetch_current_tenant(
name: &tenant.display_name,
active: true,
})
.execute(conn)?;
.execute(db_conn)?;
} else if diesel::select(diesel::dsl::exists(
db::schema::tenants::table
.filter(db::schema::tenants::untis_id.eq(tenant.id))
.filter(db::schema::tenants::active.eq(false)),
))
.get_result::<bool>(conn)?
.get_result::<bool>(db_conn)?
{
diesel::update(db::schema::tenants::table)
.filter(db::schema::tenants::active)
@ -71,14 +74,14 @@ async fn fetch_current_tenant(
db::schema::tenants::active.eq(false),
db::schema::tenants::updated_at.eq(diesel::dsl::now),
))
.execute(conn)?;
.execute(db_conn)?;
diesel::update(db::schema::tenants::table)
.filter(db::schema::tenants::untis_id.eq(tenant.id))
.set((
db::schema::tenants::active.eq(true),
db::schema::tenants::updated_at.eq(diesel::dsl::now),
))
.execute(conn)?;
.execute(db_conn)?;
}
Ok(())
@ -86,13 +89,13 @@ async fn fetch_current_tenant(
async fn fetch_teachers(
client: &untis::Client,
conn: &mut PgConnection,
db_conn: &mut PgConnection,
schoolyear_id: i32,
) -> Result<()> {
let existing_teachers = db::schema::teachers::table
.select(db::schema::teachers::untis_id)
.filter(db::schema::teachers::schoolyear_id.eq(schoolyear_id))
.load::<i32>(conn)?;
.load::<i32>(db_conn)?;
diesel::insert_into(db::schema::teachers::table)
.values(
&client
@ -113,20 +116,20 @@ async fn fetch_teachers(
})
.collect::<Vec<db::models::NewTeacher>>(),
)
.execute(conn)?;
.execute(db_conn)?;
Ok(())
}
async fn fetch_classes(
client: &untis::Client,
conn: &mut PgConnection,
db_conn: &mut PgConnection,
schoolyear_id: i32,
) -> Result<()> {
let existing_classes = db::schema::classes::table
.select(db::schema::classes::untis_id)
.filter(db::schema::classes::schoolyear_id.eq(schoolyear_id))
.load::<i32>(conn)?;
.load::<i32>(db_conn)?;
diesel::insert_into(db::schema::classes::table)
.values(
&client
@ -143,20 +146,20 @@ async fn fetch_classes(
})
.collect::<Vec<db::models::NewClass>>(),
)
.execute(conn)?;
.execute(db_conn)?;
Ok(())
}
async fn fetch_subjects(
client: &untis::Client,
conn: &mut PgConnection,
db_conn: &mut PgConnection,
schoolyear_id: i32,
) -> Result<()> {
let existing_classes = db::schema::subjects::table
.select(db::schema::subjects::untis_id)
.filter(db::schema::subjects::schoolyear_id.eq(schoolyear_id))
.load::<i32>(conn)?;
.load::<i32>(db_conn)?;
diesel::insert_into(db::schema::subjects::table)
.values(
&client
@ -172,20 +175,20 @@ async fn fetch_subjects(
})
.collect::<Vec<db::models::NewSubject>>(),
)
.execute(conn)?;
.execute(db_conn)?;
Ok(())
}
async fn fetch_rooms(
client: &untis::Client,
conn: &mut PgConnection,
db_conn: &mut PgConnection,
schoolyear_id: i32,
) -> Result<()> {
let existing_classes = db::schema::rooms::table
.select(db::schema::rooms::untis_id)
.filter(db::schema::rooms::schoolyear_id.eq(schoolyear_id))
.load::<i32>(conn)?;
.load::<i32>(db_conn)?;
diesel::insert_into(db::schema::rooms::table)
.values(
&client
@ -201,20 +204,20 @@ async fn fetch_rooms(
})
.collect::<Vec<db::models::NewRoom>>(),
)
.execute(conn)?;
.execute(db_conn)?;
Ok(())
}
async fn fetch_departments(
client: &untis::Client,
conn: &mut PgConnection,
db_conn: &mut PgConnection,
schoolyear_id: i32,
) -> Result<()> {
let existing_classes = db::schema::departments::table
.select(db::schema::departments::untis_id)
.filter(db::schema::departments::schoolyear_id.eq(schoolyear_id))
.load::<i32>(conn)?;
.load::<i32>(db_conn)?;
diesel::insert_into(db::schema::departments::table)
.values(
&client
@ -230,20 +233,20 @@ async fn fetch_departments(
})
.collect::<Vec<db::models::NewDepartment>>(),
)
.execute(conn)?;
.execute(db_conn)?;
Ok(())
}
async fn fetch_holidays(
client: &untis::Client,
conn: &mut PgConnection,
db_conn: &mut PgConnection,
schoolyear_id: i32,
) -> Result<()> {
let existing_classes = db::schema::holidays::table
.select(db::schema::holidays::untis_id)
.filter(db::schema::holidays::schoolyear_id.eq(schoolyear_id))
.load::<i32>(conn)?;
.load::<i32>(db_conn)?;
diesel::insert_into(db::schema::holidays::table)
.values(
&client
@ -261,147 +264,198 @@ async fn fetch_holidays(
})
.collect::<Vec<db::models::NewHoliday>>(),
)
.execute(conn)?;
.execute(db_conn)?;
Ok(())
}
lazy_static! {
static ref TITLE_SELECTOR: scraper::Selector = scraper::Selector::parse(".mon_title").unwrap();
}
async fn fetch_substitutions(
client: &untis::Client,
conn: &mut PgConnection,
db_conn: &mut PgConnection,
redis_conn: &mut cache::ConnectionPool,
schoolyear_id: i32,
) -> Result<()> {
let today = Utc::now().date_naive();
if diesel::select(diesel::dsl::not(diesel::dsl::exists(
db::schema::substitution_queries::table
.filter(db::schema::substitution_queries::date.eq(today)),
)))
.get_result::<bool>(conn)?
{
diesel::insert_into(db::schema::substitution_queries::table)
.values(db::models::NewSubstitutionQuery {
schoolyear_id,
date: today,
active: true,
let (date, week_type) = {
let html = reqwest::Client::new()
.get(&config::CONFIG.untis_vplan_url)
.header(
reqwest::header::USER_AGENT,
&config::CONFIG.untis_client_name,
)
.header(reqwest::header::ACCEPT, "text/html")
.basic_auth(
&config::CONFIG.untis_vplan_username,
Some(&config::CONFIG.untis_vplan_password),
)
.send()
.await?
.text()
.await?;
let document = scraper::Html::parse_document(&html);
let title = document
.select(&TITLE_SELECTOR)
.next()
.context("No element in vplan html which is selectable class \"mon_title\"")?
.text()
.next()
.context("\"mon_title\" element is empty")?
.split_once(',')
.context("Could not split \"mon_title\" string")?;
(
chrono::NaiveDate::parse_from_str(
title
.0
.split_whitespace()
.next()
.context("Could not find date")?,
"%d.%m.%Y",
)?,
match title
.1
.split_whitespace()
.last()
.context("Could not find week type indicator")?
{
"A" => db::models::WeekType::A,
"B" => db::models::WeekType::B,
x => bail!("Invalid week type: {:?}", x),
},
)
};
let substitution_query_id = diesel::insert_into(db::schema::substitution_queries::table)
.values(db::models::NewSubstitutionQuery {
schoolyear_id,
date,
week_type,
queried_at: Utc::now().naive_utc(),
})
.returning(db::schema::substitution_queries::id)
.get_result::<i32>(db_conn)?;
redis::cmd("SET")
.arg(cache::keys::LAST_SUBSTITUTION_QUERY_ID)
.arg(substitution_query_id)
.query(redis_conn.deref_mut())?;
for substitution in client.substitutions(&date, &date, None).await? {
let substitution_id = diesel::insert_into(db::schema::substitutions::table)
.values(db::models::NewSubstitution {
substitution_query_id,
subst_type: substitution.subst_type.into(),
lesson_id: substitution.lesson_id,
start_time: substitution.start_time,
end_time: substitution.end_time,
text: substitution.text.as_deref(),
})
.execute(conn)?;
}
.returning(db::schema::substitutions::id)
.get_result::<i32>(db_conn)?;
for query in db::schema::substitution_queries::table
.filter(db::schema::substitution_queries::active)
.load::<db::models::SubstitutionQuery>(conn)?
{
let query_result_id = diesel::insert_into(db::schema::substitution_query_results::table)
.values(db::models::NewSubstitutionQueryResult {
substitution_query_id: query.id,
queried_at: Utc::now().naive_utc(),
})
.returning(db::schema::substitution_query_results::id)
.get_result::<i32>(conn)?;
diesel::insert_into(db::schema::substitution_classes::table)
.values(
&substitution
.classes
.iter()
.enumerate()
.map(|(i, c)| {
Ok(db::models::NewSubstitutionClass {
substitution_id,
position: i as i16,
class_id: db::schema::classes::table
.filter(db::schema::classes::untis_id.eq(c.id))
.select(db::schema::classes::id)
.get_result::<i32>(db_conn)?,
})
})
.collect::<Result<Vec<db::models::NewSubstitutionClass>>>()?,
)
.execute(db_conn)?;
let substs = client.substitutions(&query.date, &query.date, None).await?;
for substitution in substs {
let substitution_id = diesel::insert_into(db::schema::substitutions::table)
.values(db::models::NewSubstitution {
substitution_query_result_id: query_result_id,
subst_type: substitution.subst_type.into(),
lesson_id: substitution.lesson_id,
start_time: substitution.start_time,
end_time: substitution.end_time,
text: substitution.text.as_deref(),
})
.returning(db::schema::substitutions::id)
.get_result::<i32>(conn)?;
diesel::insert_into(db::schema::substitution_teachers::table)
.values(
&substitution
.teachers
.iter()
.enumerate()
.map(|(i, t)| {
Ok(db::models::NewSubstitutionTeacher {
substitution_id,
position: i as i16,
teacher_id: if t.id == 0 {
None
} else {
Some(
db::schema::teachers::table
.filter(db::schema::teachers::untis_id.eq(t.id))
.select(db::schema::teachers::id)
.get_result::<i32>(db_conn)?,
)
},
})
})
.collect::<Result<Vec<db::models::NewSubstitutionTeacher>>>()?,
)
.execute(db_conn)?;
diesel::insert_into(db::schema::substitution_classes::table)
.values(
&substitution
.classes
.iter()
.enumerate()
.map(|(i, c)| {
Ok(db::models::NewSubstitutionClass {
substitution_id,
position: i as i16,
class_id: db::schema::classes::table
.filter(db::schema::classes::untis_id.eq(c.id))
.select(db::schema::classes::id)
.get_result::<i32>(conn)?,
})
diesel::insert_into(db::schema::substitution_subjects::table)
.values(
&substitution
.subjects
.iter()
.enumerate()
.map(|(i, s)| {
Ok(db::models::NewSubstitutionSubject {
substitution_id,
position: i as i16,
subject_id: db::schema::subjects::table
.filter(db::schema::subjects::untis_id.eq(s.id))
.select(db::schema::subjects::id)
.get_result::<i32>(db_conn)?,
})
.collect::<Result<Vec<db::models::NewSubstitutionClass>>>()?,
)
.execute(conn)?;
diesel::insert_into(db::schema::substitution_teachers::table)
.values(
&substitution
.teachers
.iter()
.enumerate()
.map(|(i, t)| {
Ok(db::models::NewSubstitutionTeacher {
substitution_id,
position: i as i16,
teacher_id: if t.id == 0 {
None
} else {
Some(
db::schema::teachers::table
.filter(db::schema::teachers::untis_id.eq(t.id))
.select(db::schema::teachers::id)
.get_result::<i32>(conn)?,
)
},
})
})
.collect::<Result<Vec<db::models::NewSubstitutionSubject>>>()?,
)
.execute(db_conn)?;
diesel::insert_into(db::schema::substitution_rooms::table)
.values(
&substitution
.rooms
.iter()
.enumerate()
.map(|(i, r)| {
Ok(db::models::NewSubstitutionRoom {
substitution_id,
position: i as i16,
room_id: if r.id == 0 {
None
} else {
Some(
db::schema::rooms::table
.filter(db::schema::rooms::untis_id.eq(r.id))
.select(db::schema::rooms::id)
.get_result::<i32>(db_conn)?,
)
},
original_room_id: if let Some(original_id) = r.original_id {
Some(
db::schema::rooms::table
.filter(db::schema::rooms::untis_id.eq(original_id))
.select(db::schema::rooms::id)
.get_result::<i32>(db_conn)?,
)
} else {
None
},
})
.collect::<Result<Vec<db::models::NewSubstitutionTeacher>>>()?,
)
.execute(conn)?;
diesel::insert_into(db::schema::substitution_subjects::table)
.values(
&substitution
.subjects
.iter()
.enumerate()
.map(|(i, s)| {
Ok(db::models::NewSubstitutionSubject {
substitution_id,
position: i as i16,
subject_id: db::schema::subjects::table
.filter(db::schema::subjects::untis_id.eq(s.id))
.select(db::schema::subjects::id)
.get_result::<i32>(conn)?,
})
})
.collect::<Result<Vec<db::models::NewSubstitutionSubject>>>()?,
)
.execute(conn)?;
diesel::insert_into(db::schema::substitution_rooms::table)
.values(
&substitution
.rooms
.iter()
.enumerate()
.map(|(i, r)| {
Ok(db::models::NewSubstitutionRoom {
substitution_id,
position: i as i16,
room_id: if r.id == 0 {
None
} else {
Some(
db::schema::rooms::table
.filter(db::schema::rooms::untis_id.eq(r.id))
.select(db::schema::rooms::id)
.get_result::<i32>(conn)?,
)
},
})
})
.collect::<Result<Vec<db::models::NewSubstitutionRoom>>>()?,
)
.execute(conn)?;
}
})
.collect::<Result<Vec<db::models::NewSubstitutionRoom>>>()?,
)
.execute(db_conn)?;
}
Ok(())
@ -409,10 +463,9 @@ async fn fetch_substitutions(
#[celery::task]
pub async fn update_info() -> TaskResult<()> {
let dur = Duration::from_secs(10);
let dur = Duration::from_secs(2);
thread::sleep(dur);
dbg!("DONE!");
let mut client = match config::untis_from_env() {
Ok(x) => x,
Err(e) => return Err(TaskError::UnexpectedError(e.to_string())),
@ -421,65 +474,58 @@ pub async fn update_info() -> TaskResult<()> {
return Err(TaskError::UnexpectedError(e.to_string()));
}
thread::sleep(dur);
dbg!("DONE!");
let conn = &mut match db::POOL.get() {
let db_conn = &mut match db::POOL.get() {
Ok(x) => x,
Err(e) => return Err(TaskError::UnexpectedError(e.to_string())),
};
let redis_conn = &mut match cache::POOL.get() {
Ok(x) => x,
Err(e) => return Err(TaskError::UnexpectedError(e.to_string())),
};
let schoolyear_id = match fetch_schoolyears(&client, conn).await {
let schoolyear_id = match fetch_schoolyears(&client, db_conn).await {
Ok(x) => x,
Err(e) => return Err(TaskError::UnexpectedError(e.to_string())),
};
thread::sleep(dur);
dbg!("DONE!");
if let Err(e) = fetch_current_tenant(&client, conn, schoolyear_id).await {
if let Err(e) = fetch_current_tenant(&client, db_conn, schoolyear_id).await {
return Err(TaskError::UnexpectedError(e.to_string()));
}
thread::sleep(dur);
dbg!("DONE!");
if let Err(e) = fetch_teachers(&client, conn, schoolyear_id).await {
if let Err(e) = fetch_teachers(&client, db_conn, schoolyear_id).await {
return Err(TaskError::UnexpectedError(e.to_string()));
}
thread::sleep(dur);
dbg!("DONE!");
if let Err(e) = fetch_classes(&client, conn, schoolyear_id).await {
if let Err(e) = fetch_classes(&client, db_conn, schoolyear_id).await {
return Err(TaskError::UnexpectedError(e.to_string()));
}
thread::sleep(dur);
dbg!("DONE!");
if let Err(e) = fetch_subjects(&client, conn, schoolyear_id).await {
if let Err(e) = fetch_subjects(&client, db_conn, schoolyear_id).await {
return Err(TaskError::UnexpectedError(e.to_string()));
}
thread::sleep(dur);
dbg!("DONE!");
if let Err(e) = fetch_rooms(&client, conn, schoolyear_id).await {
if let Err(e) = fetch_rooms(&client, db_conn, schoolyear_id).await {
return Err(TaskError::UnexpectedError(e.to_string()));
}
thread::sleep(dur);
dbg!("DONE!");
if let Err(e) = fetch_departments(&client, conn, schoolyear_id).await {
if let Err(e) = fetch_departments(&client, db_conn, schoolyear_id).await {
return Err(TaskError::UnexpectedError(e.to_string()));
}
thread::sleep(dur);
dbg!("DONE!");
if let Err(e) = fetch_holidays(&client, conn, schoolyear_id).await {
if let Err(e) = fetch_holidays(&client, db_conn, schoolyear_id).await {
return Err(TaskError::UnexpectedError(e.to_string()));
}
thread::sleep(dur);
dbg!("DONE!");
if let Err(e) = fetch_substitutions(&client, conn, schoolyear_id).await {
if let Err(e) = fetch_substitutions(&client, db_conn, redis_conn, schoolyear_id).await {
return Err(TaskError::UnexpectedError(e.to_string()));
}
thread::sleep(dur);
dbg!("DONE!");
if let Err(e) = client.logout().await {
return Err(TaskError::UnexpectedError(e.to_string()));
}
thread::sleep(dur);
dbg!("DONE!");
Ok(())
}