leon/knifer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switch to custom demo parser

Browse Source
This commit is contained in:
Lol3rrr
2024-09-15 23:18:43 +02:00
parent 0ce8da7286
commit 44020a408f
5 changed files with 153 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files

247
Cargo.lock generated
View File

@@ -17,19 +17,6 @@ version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
[[package]]
name = "ahash"
version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
dependencies = [
"cfg-if",
"getrandom",
"once_cell",
"version_check",
"zerocopy",
]
[[package]]
name = "aho-corasick"
version = "1.1.3"
@@ -41,9 +28,9 @@ dependencies = [
[[package]]
name = "anyhow"
version = "1.0.88"
version = "1.0.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e1496f8fb1fbf272686b8d37f523dab3e4a7443300055e74cdaa449f3114356"
checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6"
[[package]]
name = "async-recursion"
@@ -169,16 +156,15 @@ dependencies = [
name = "backend"
version = "0.1.0"
dependencies = [
"ahash",
"async-trait",
"axum",
"common",
"csdemo",
"diesel",
"diesel-async",
"diesel_async_migrations",
"futures-util",
"memmap2",
"parser",
"reqwest 0.12.7",
"serde",
"serde_json",
@@ -218,12 +204,6 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "bit_reverse"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99528ca30abb9495c7e106bf7c3177b257c62040fc0f2909fe470b0f43097296"
[[package]]
name = "bitflags"
version = "1.3.2"
@@ -336,7 +316,7 @@ version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7328b20597b53c2454f0b1919720c25c7339051c02b72b7e05409e00b14132be"
dependencies = [
"convert_case 0.6.0",
"convert_case",
"lazy_static",
"nom",
"pathdiff",
@@ -364,12 +344,6 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "convert_case"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"
[[package]]
name = "convert_case"
version = "0.6.0"
@@ -415,31 +389,6 @@ dependencies = [
"libc",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
[[package]]
name = "crunchy"
version = "0.2.2"
@@ -457,14 +406,16 @@ dependencies = [
]
[[package]]
name = "csgoproto"
version = "0.1.5"
source = "git+https://github.com/LaihoE/demoparser.git#60974266a0374000f83d31172ac823135e93482b"
name = "csdemo"
version = "0.1.0"
source = "git+https://github.com/Lol3rrr/csdemo.git#4417b263a0c04ceb32e199e67dd790adf4645277"
dependencies = [
"bytes",
"glob",
"proc-macro2",
"protobuf",
"bitter",
"phf",
"prost",
"prost-build",
"prost-types",
"snap",
]
[[package]]
@@ -536,19 +487,6 @@ dependencies = [
"syn",
]
[[package]]
name = "derive_more"
version = "0.99.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce"
dependencies = [
"convert_case 0.4.0",
"proc-macro2",
"quote",
"rustc_version",
"syn",
]
[[package]]
name = "diesel"
version = "2.2.4"
@@ -695,6 +633,12 @@ version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6"
[[package]]
name = "fixedbitset"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
[[package]]
name = "fnv"
version = "1.0.7"
@@ -854,12 +798,6 @@ version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64"
[[package]]
name = "glob"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
[[package]]
name = "gloo-net"
version = "0.1.0"
@@ -1368,7 +1306,7 @@ checksum = "4b13bc3db70715cd8218c4535a5af3ae3c0e5fea6f018531fc339377b36bc0e0"
dependencies = [
"attribute-derive",
"cfg-if",
"convert_case 0.6.0",
"convert_case",
"html-escape",
"itertools 0.12.1",
"leptos_hot_reload",
@@ -1553,9 +1491,9 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
name = "memmap2"
version = "0.9.4"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322"
checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f"
dependencies = [
"libc",
]
@@ -1620,6 +1558,12 @@ dependencies = [
"version_check",
]
[[package]]
name = "multimap"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03"
[[package]]
name = "native-tls"
version = "0.2.12"
@@ -1684,9 +1628,9 @@ dependencies = [
[[package]]
name = "once_cell"
version = "1.19.0"
version = "1.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
checksum = "33ea5043e58958ee56f3e15a90aee535795cd7dfd319846288d93c5b57d85cbe"
[[package]]
name = "openssl"
@@ -1767,33 +1711,6 @@ dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "parser"
version = "0.1.1"
source = "git+https://github.com/LaihoE/demoparser.git#60974266a0374000f83d31172ac823135e93482b"
dependencies = [
"ahash",
"bit_reverse",
"bitter",
"bytes",
"csgoproto",
"derive_more",
"itertools 0.13.0",
"lazy_static",
"libc",
"memmap2",
"phf",
"phf_macros",
"proc-macro2",
"protobuf",
"protobuf-support",
"rand",
"rayon",
"regex",
"serde",
"snap",
]
[[package]]
name = "paste"
version = "1.0.15"
@@ -1812,12 +1729,23 @@ version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
[[package]]
name = "petgraph"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db"
dependencies = [
"fixedbitset",
"indexmap",
]
[[package]]
name = "phf"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc"
dependencies = [
"phf_macros",
"phf_shared",
]
@@ -2034,24 +1962,56 @@ dependencies = [
]
[[package]]
name = "protobuf"
version = "3.5.0"
name = "prost"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df67496db1a89596beaced1579212e9b7c53c22dca1d9745de00ead76573d514"
checksum = "3b2ecbe40f08db5c006b5764a2645f7f3f141ce756412ac9e1dd6087e6d32995"
dependencies = [
"bytes",
"once_cell",
"protobuf-support",
"thiserror",
"prost-derive",
]
[[package]]
name = "protobuf-support"
version = "3.5.0"
name = "prost-build"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70e2d30ab1878b2e72d1e2fc23ff5517799c9929e2cf81a8516f9f4dcf2b9cf3"
checksum = "f8650aabb6c35b860610e9cff5dc1af886c9e25073b7b1712a68972af4281302"
dependencies = [
"thiserror",
"bytes",
"heck",
"itertools 0.13.0",
"log",
"multimap",
"once_cell",
"petgraph",
"prettyplease",
"prost",
"prost-types",
"regex",
"syn",
"tempfile",
]
[[package]]
name = "prost-derive"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acf0c195eebb4af52c752bec4f52f645da98b6e92077a04110c7f349477ae5ac"
dependencies = [
"anyhow",
"itertools 0.13.0",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "prost-types"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60caa6738c7369b940c3d49246a8d1749323674c65cb13010134f5c9bad5b519"
dependencies = [
"prost",
]
[[package]]
@@ -2115,26 +2075,6 @@ dependencies = [
"getrandom",
]
[[package]]
name = "rayon"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
dependencies = [
"crossbeam-deque",
"crossbeam-utils",
]
[[package]]
name = "redox_syscall"
version = "0.5.4"
@@ -2306,15 +2246,6 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustc_version"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
dependencies = [
"semver",
]
[[package]]
name = "rustix"
version = "0.38.37"
@@ -2452,12 +2383,6 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d369a96f978623eb3dc28807c4852d6cc617fed53da5d3c400feff1ef34a714a"
[[package]]
name = "semver"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
[[package]]
name = "send_wrapper"
version = "0.6.0"
@@ -2608,7 +2533,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "faaaf648c6967aef78177c0610478abb5a3455811f401f3c62d10ae9bd3901a1"
dependencies = [
"const_format",
"convert_case 0.6.0",
"convert_case",
"proc-macro2",
"quote",
"syn",
@@ -3292,9 +3217,9 @@ checksum = "52ea75f83c0137a9b98608359a5f1af8144876eb67bcb1ce837368e906a9f524"
[[package]]
name = "unicode-segmentation"
version = "1.11.0"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
[[package]]
name = "unicode-xid"

3
backend/Cargo.toml
View File

@@ -25,6 +25,5 @@ reqwest = { version = "0.12", features = ["json"] }
common = { path = "../common/" }
l_demoparser = { package = "parser", git = "https://github.com/LaihoE/demoparser.git", ref = "main" }
ahash = { version = "0.8" }
csdemo = { package = "csdemo", git = "https://github.com/Lol3rrr/csdemo.git", ref = "main" }
memmap2 = { version = "0.9" }

109
backend/src/analysis.rs
View File

@@ -17,62 +17,44 @@ pub struct BaseInfo {
pub fn analyse_base(input: AnalysisInput) -> BaseInfo {
tracing::info!("Performing Base analysis");
let huf = l_demoparser::second_pass::parser_settings::create_huffman_lookup_table();
let settings = l_demoparser::first_pass::parser_settings::ParserInputs {
wanted_players: Vec::new(),
real_name_to_og_name: ahash::AHashMap::default(),
wanted_player_props: vec!["X".to_string(), "team_num".to_string()],
wanted_events: vec!["player_death".to_string(), "player_team".to_string(), "team_info".to_string(), "player_spawn".to_string(), "team_score".to_string(), "round_end".to_string(), "game_end".to_string(), "match_end_conditions".to_string(), "switch_team".to_string(), "player_given_c4".to_string()],
wanted_other_props: vec![],
parse_ents: true,
wanted_ticks: Vec::new(),
parse_projectiles: false,
only_header: false,
count_props: false,
only_convars: false,
huffman_lookup_table: &huf,
order_by_steamid: false,
};
let mut ds = l_demoparser::parse_demo::Parser::new(settings, l_demoparser::parse_demo::ParsingMode::ForceSingleThreaded);
let file = std::fs::File::open(&input.path).unwrap();
let mmap = unsafe { memmap2::MmapOptions::new().map(&file).unwrap() };
let output = ds.parse_demo(&mmap).unwrap();
let header = output.header.as_ref().unwrap();
let tmp = csdemo::Container::parse(&mmap).unwrap();
let output = csdemo::parser::parse(csdemo::FrameIterator::parse(tmp.inner)).unwrap();
let header = &output.header;
tracing::info!("Header: {:?}", header);
for event in output.game_events.iter() {
match event.name.as_str() {
"player_team" => {
let team = event.fields.iter().find_map(|f| match (f.name.as_str(), &f.data) {
("team", Some(d)) => Some(d),
("team", None) => {
tracing::warn!("'team' field without data");
None
}
_ => None,
});
let user_name = event.fields.iter().find_map(|f| match (f.name.as_str(), &f.data) {
("user_name", Some(d)) => Some(d),
("user_name", None) => {
tracing::warn!("'user_name' field without data");
None
}
_ => None,
});
let steamid = event.fields.iter().find_map(|f| match (f.name.as_str(), &f.data) {
("user_steamid", Some(d)) => Some(d),
("user_steamid", None) => {
tracing::warn!("'user_steamid' field without data");
None
}
_ => None,
});
dbg!(&output.player_info);
tracing::info!("'{:?}' ({:?}) -> {:?}", user_name, steamid, team);
for event in output.events.iter() {
match event {
csdemo::DemoEvent::Tick(tick) => {}
csdemo::DemoEvent::ServerInfo(info) => {}
csdemo::DemoEvent::RankUpdate(update) => {}
csdemo::DemoEvent::RankReveal(reveal) => {}
csdemo::DemoEvent::GameEvent(gevent) => {
match gevent {
csdemo::game_event::GameEvent::PlayerTeam(pteam) => {
tracing::info!("{:?}", pteam);
}
csdemo::game_event::GameEvent::RoundOfficiallyEnded(r_end) => {
tracing::info!("{:?}", r_end);
}
csdemo::game_event::GameEvent::PlayerDeath(pdeath) => {
tracing::info!("{:?}", pdeath);
}
other => {}
};
}
};
/*
match event.name.as_str() {
"team_info" => {
tracing::info!("Team Info: {:?}", event);
}
@@ -82,34 +64,6 @@ pub fn analyse_base(input: AnalysisInput) -> BaseInfo {
"team_score" => {
tracing::info!("Team Score: {:?}", event);
}
"round_end" => {
let winner = event.fields.iter().find_map(|f| match (f.name.as_str(), &f.data) {
("winner", Some(d)) => Some(d),
("winner", None) => {
tracing::warn!("'winner' field without data");
None
}
_ => None,
});
let round = event.fields.iter().find_map(|f| match (f.name.as_str(), &f.data) {
("round", Some(d)) => Some(d),
("round", None) => {
tracing::warn!("'round' field without data");
None
}
_ => None,
});
let reason = event.fields.iter().find_map(|f| match (f.name.as_str(), &f.data) {
("reason", Some(d)) => Some(d),
("reason", None) => {
tracing::warn!("'reason' field without data");
None
}
_ => None,
});
tracing::info!(?winner, ?round, ?reason, "Round End");
}
"game_end" => {
tracing::info!("Game End: {:?}", event);
}
@@ -124,10 +78,13 @@ pub fn analyse_base(input: AnalysisInput) -> BaseInfo {
}
_ => {}
};
*/
}
let map = header.map_name().to_owned();
BaseInfo {
map: header.get("map_name").cloned().unwrap_or_default(),
map,
players: Vec::new()
}
}

26
backend/src/api/demos.rs
View File

@@ -14,6 +14,7 @@ use crate::UserSession;
.route("/list", axum::routing::get(list))
.route("/upload", axum::routing::post(upload).layer(axum::extract::DefaultBodyLimit::max(500*1024*1024)))
.route("/:id/info", axum::routing::get(info))
.route("/:id/reanalyse", axum::routing::get(analyise))
.with_state(Arc::new(DemoState {
upload_folder: upload_folder.into(),
base_analysis,
@@ -75,6 +76,31 @@ use crate::UserSession;
Ok(axum::response::Redirect::to("/"))
}
#[tracing::instrument(skip(state, session))]
async fn analyise(State(state): State<Arc<DemoState>>, session: crate::UserSession, Path(demo_id): Path<i64>) -> Result<(), (axum::http::StatusCode, &'static str)> {
let steam_id = session.data().steam_id.ok_or_else(|| (axum::http::StatusCode::UNAUTHORIZED, "Not logged in"))?;
tracing::info!("Upload for Session: {:?}", steam_id);
let mut db_con = crate::db_connection().await;
let query = crate::schema::demos::dsl::demos.filter(crate::schema::demos::dsl::steam_id.eq(steam_id.to_string())).filter(crate::schema::demos::dsl::demo_id.eq(demo_id));
let result: Vec<_> = query.load::<crate::models::Demo>(&mut db_con).await.unwrap();
if result.len() != 1 {
return Err((axum::http::StatusCode::BAD_REQUEST, "Expected exactly 1 demo to match"));
}
let user_folder = std::path::Path::new(&state.upload_folder).join(format!("{}/", steam_id));
state.base_analysis.send(crate::analysis::AnalysisInput {
path: user_folder.join(format!("{}.dem", demo_id)),
demoid: demo_id,
steamid: steam_id.to_string(),
});
Ok(())
}
#[tracing::instrument(skip(session))]
async fn info(session: UserSession, Path(demo_id): Path<i64>) -> Result<axum::response::Json<common::DemoInfo>, axum::http::StatusCode> {
tracing::info!("Get info for Demo: {:?}", demo_id);

9
backend/src/main.rs
View File

@@ -28,6 +28,8 @@ async fn main() {
let (base_analysis_tx, mut base_analysis_rx) = tokio::sync::mpsc::unbounded_channel::<backend::analysis::AnalysisInput>();
tokio::task::spawn_blocking(move || {
use diesel_async::AsyncConnection;
while let Some(input) = base_analysis_rx.blocking_recv() {
let demo_id = input.demoid;
let result = backend::analysis::analyse_base(input);
@@ -47,8 +49,11 @@ async fn main() {
tracing::trace!(?store_info_query, "Store demo info query");
tracing::trace!(?update_process_info, "Update processing info query");
store_info_query.execute(&mut db_con).await.unwrap();
update_process_info.execute(&mut db_con).await.unwrap();
db_con.transaction::<'_, '_, '_, _, diesel::result::Error, _>(|conn| Box::pin(async move {
store_info_query.execute(conn).await.map(|e| ())?;
update_process_info.execute(conn).await.map(|e| ())?;
Ok(())
})).await;
}
);
}