refactor: implement packet parsing heuristics and introduce device-level state caching to throttle database writes and vitals updates
This commit is contained in:
@@ -4,6 +4,7 @@ from typing import Optional, List
|
||||
import struct
|
||||
|
||||
from schemas import NormalizedVitals
|
||||
from config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -79,7 +80,7 @@ def safe_float(value: str) -> Optional[float]:
|
||||
|
||||
|
||||
def is_valid_u16(val: Optional[int]) -> bool:
|
||||
return val is not None and val != INVALID_SENTINEL_U16
|
||||
return val is not None and val != INVALID_SENTINEL_U16 and val != 255 and val != 65535
|
||||
|
||||
|
||||
def is_valid_f32(val: Optional[float]) -> bool:
|
||||
@@ -118,14 +119,16 @@ def parse_contec_hl7_text(raw_text: str, client_ip: str) -> Optional[NormalizedV
|
||||
return None
|
||||
|
||||
# Parse MSH segment
|
||||
model = settings.device_models.get(client_ip, settings.monitor_model)
|
||||
msh_fields = segments[0].split("|")
|
||||
device_id = msh_fields[2] if len(msh_fields) > 2 and msh_fields[2] else f"CMS7000_{client_ip}"
|
||||
device_id = msh_fields[2] if len(msh_fields) > 2 and msh_fields[2] else f"{model}_{client_ip}"
|
||||
|
||||
patient_id = "UNKNOWN"
|
||||
vitals_dict = {
|
||||
"device_id": device_id,
|
||||
"patient_id": patient_id,
|
||||
"timestamp": datetime.now(timezone.utc),
|
||||
"ip_address": client_ip,
|
||||
}
|
||||
|
||||
found_any = False
|
||||
@@ -145,7 +148,6 @@ def parse_contec_hl7_text(raw_text: str, client_ip: str) -> Optional[NormalizedV
|
||||
|
||||
elif seg_name == "OBX":
|
||||
if len(fields) > 5:
|
||||
obs_id = fields[3].split("^")[0].strip().upper()
|
||||
obs_val = fields[5].strip()
|
||||
|
||||
if not obs_val or obs_val == "---":
|
||||
@@ -154,12 +156,20 @@ def parse_contec_hl7_text(raw_text: str, client_ip: str) -> Optional[NormalizedV
|
||||
val_float = safe_float(obs_val)
|
||||
if val_float is None:
|
||||
continue
|
||||
|
||||
|
||||
# Split OBX-3 (Observation Identifier) by '^' and check all parts for parameter matches
|
||||
obs_parts = [p.strip().upper() for p in fields[3].split("^") if p.strip()]
|
||||
|
||||
matched = False
|
||||
for param_key, mapped_field in CONTEC_PARAM_MAP.items():
|
||||
if obs_id == param_key or param_key in obs_id:
|
||||
vitals_dict[mapped_field] = val_float
|
||||
found_any = True
|
||||
logger.info(f"Parsed Contec HL7 Vital: {mapped_field} = {val_float} (from {obs_id})")
|
||||
for part in obs_parts:
|
||||
if param_key == part or param_key in part:
|
||||
vitals_dict[mapped_field] = val_float
|
||||
found_any = True
|
||||
logger.info(f"Parsed Contec HL7 Vital: {mapped_field} = {val_float} (from OBX-3: {fields[3]})")
|
||||
matched = True
|
||||
break
|
||||
if matched:
|
||||
break
|
||||
|
||||
return NormalizedVitals(**vitals_dict) if found_any else None
|
||||
@@ -192,10 +202,12 @@ def _extract_vitals_block(data: bytes, offset: int, client_ip: str) -> Optional[
|
||||
Offset +10 : uint16 LE → NIBP Systolic (9999 = invalid)
|
||||
Offset +12 : uint16 LE → NIBP Diastolic (9999 = invalid)
|
||||
"""
|
||||
model = settings.device_models.get(client_ip, settings.monitor_model)
|
||||
vitals_dict = {
|
||||
"device_id": f"CMS7000PLUS_{client_ip}",
|
||||
"device_id": f"{model}_{client_ip}",
|
||||
"patient_id": "UNKNOWN",
|
||||
"timestamp": datetime.now(timezone.utc),
|
||||
"ip_address": client_ip,
|
||||
}
|
||||
found = False
|
||||
|
||||
@@ -219,113 +231,271 @@ def _extract_vitals_block(data: bytes, offset: int, client_ip: str) -> Optional[
|
||||
return NormalizedVitals(**vitals_dict) if found else None
|
||||
|
||||
|
||||
def _parse_float_vitals(data: bytes, base_offset: int, vitals_dict: dict) -> bool:
|
||||
def _parse_45_byte_packet(data: bytes, vitals_dict: dict) -> bool:
|
||||
"""
|
||||
Parse float32 vitals (NIBP, Temperature) from packet starting at base_offset.
|
||||
Returns True if any valid vitals found.
|
||||
Parse 45-byte packet (Subtype 22) — contains NIBP values and alarm limits.
|
||||
|
||||
Verified layout:
|
||||
[0-1] Packet length LE u16 = 45
|
||||
[2-3] Marker: 01 46
|
||||
[4-7] Sub-header (00 00 16 00)
|
||||
[8-9] Year LE u16
|
||||
[10] Month, [11] Day, [12] Hour, [13] Minute, [14] Seconds
|
||||
[15-16] NIBP Systolic (LE u16, 9999 = no measurement)
|
||||
[17-18] NIBP Diastolic (LE u16, 9999 = no measurement)
|
||||
[19-20] NIBP MAP (LE u16, 9999 = no measurement)
|
||||
"""
|
||||
if len(data) < 22:
|
||||
return False
|
||||
|
||||
if "present_fields" not in vitals_dict:
|
||||
vitals_dict["present_fields"] = []
|
||||
|
||||
nibp_sys = read_u16_le(data, 15)
|
||||
nibp_dia = read_u16_le(data, 17)
|
||||
nibp_map = read_u16_le(data, 19)
|
||||
|
||||
found = False
|
||||
|
||||
nibp_sys = read_f32_le(data, base_offset)
|
||||
nibp_dia = read_f32_le(data, base_offset + 4)
|
||||
nibp_map = read_f32_le(data, base_offset + 8)
|
||||
temp1 = read_f32_le(data, base_offset + 12)
|
||||
temp2 = read_f32_le(data, base_offset + 16)
|
||||
# Mark NIBP fields as present so cache can clear them if 9999
|
||||
for f in ["systolic_bp", "diastolic_bp", "map_bp"]:
|
||||
if f not in vitals_dict["present_fields"]:
|
||||
vitals_dict["present_fields"].append(f)
|
||||
|
||||
if nibp_sys is not None and is_valid_f32(nibp_sys) and 50.0 < nibp_sys < 300.0:
|
||||
vitals_dict["systolic_bp"] = round(nibp_sys, 1)
|
||||
if nibp_sys is not None and nibp_sys != 9999 and 40 < nibp_sys < 250:
|
||||
vitals_dict["systolic_bp"] = float(nibp_sys)
|
||||
found = True
|
||||
else:
|
||||
vitals_dict["systolic_bp"] = None
|
||||
|
||||
if nibp_dia is not None and is_valid_f32(nibp_dia) and 20.0 < nibp_dia < 200.0:
|
||||
vitals_dict["diastolic_bp"] = round(nibp_dia, 1)
|
||||
if nibp_dia is not None and nibp_dia != 9999 and 20 < nibp_dia < 200:
|
||||
vitals_dict["diastolic_bp"] = float(nibp_dia)
|
||||
found = True
|
||||
else:
|
||||
vitals_dict["diastolic_bp"] = None
|
||||
|
||||
if nibp_map is not None and is_valid_f32(nibp_map) and 20.0 < nibp_map < 250.0:
|
||||
vitals_dict["map_bp"] = round(nibp_map, 1)
|
||||
if nibp_map is not None and nibp_map != 9999 and 20 < nibp_map < 250:
|
||||
vitals_dict["map_bp"] = float(nibp_map)
|
||||
found = True
|
||||
else:
|
||||
vitals_dict["map_bp"] = None
|
||||
|
||||
if temp1 is not None and is_valid_f32(temp1) and 30.0 < temp1 < 45.0:
|
||||
return found
|
||||
|
||||
|
||||
def _parse_56_byte_packet(data: bytes, vitals_dict: dict) -> bool:
|
||||
"""
|
||||
Parse 56-byte packet (Subtype 23) — contains float32 Temperature values.
|
||||
|
||||
Verified layout:
|
||||
[8-11] Temperature 1 (float32)
|
||||
[12-15] Temperature 2 (float32)
|
||||
[20-23] Temp 1 Alarm High Limit (float32)
|
||||
[24-27] Temp 1 Alarm Low Limit (float32)
|
||||
"""
|
||||
if len(data) < 16:
|
||||
return False
|
||||
|
||||
if "present_fields" not in vitals_dict:
|
||||
vitals_dict["present_fields"] = []
|
||||
if "temperature" not in vitals_dict["present_fields"]:
|
||||
vitals_dict["present_fields"].append("temperature")
|
||||
vitals_dict.setdefault("temperature", None)
|
||||
|
||||
found = False
|
||||
|
||||
temp1 = read_f32_le(data, 8)
|
||||
temp2 = read_f32_le(data, 12)
|
||||
|
||||
# 9999.0 is sentinel
|
||||
if temp1 is not None and is_valid_f32(temp1) and 10.0 < temp1 < 50.0:
|
||||
vitals_dict["temperature"] = round(temp1, 1)
|
||||
found = True
|
||||
elif temp2 is not None and is_valid_f32(temp2) and 30.0 < temp2 < 45.0:
|
||||
elif temp2 is not None and is_valid_f32(temp2) and 10.0 < temp2 < 50.0:
|
||||
vitals_dict["temperature"] = round(temp2, 1)
|
||||
found = True
|
||||
|
||||
return found
|
||||
|
||||
|
||||
def _parse_286_byte_packet(data: bytes, vitals_dict: dict) -> bool:
|
||||
"""
|
||||
Parse 286-byte waveform packet (Subtype 21).
|
||||
|
||||
Verified layout at tail:
|
||||
[264-265] SpO2% (LE u16) — live oxygen saturation percentage
|
||||
255 / 65535 = sensor disconnected
|
||||
[266-267] ECG HR via SpO2 PR (LE u16) — 9999 / 65535 = not available
|
||||
"""
|
||||
if len(data) < 270:
|
||||
return False
|
||||
|
||||
if "present_fields" not in vitals_dict:
|
||||
vitals_dict["present_fields"] = []
|
||||
if "spo2" not in vitals_dict["present_fields"]:
|
||||
vitals_dict["present_fields"].append("spo2")
|
||||
vitals_dict.setdefault("spo2", None)
|
||||
|
||||
found = False
|
||||
|
||||
# Offset 264: SpO2 percentage
|
||||
spo2_val = read_u16_le(data, 264)
|
||||
if spo2_val is not None and spo2_val != 65535 and spo2_val != 255 and 50 <= spo2_val <= 100:
|
||||
vitals_dict["spo2"] = float(spo2_val)
|
||||
found = True
|
||||
|
||||
# Offset 266: HR from SpO2 PR (only add to present fields if valid to prevent overriding ECG HR)
|
||||
hr_val = read_u16_le(data, 266)
|
||||
if hr_val is not None and hr_val != 65535 and hr_val != 9999 and 20 <= hr_val <= 300:
|
||||
vitals_dict["heart_rate"] = float(hr_val)
|
||||
if "heart_rate" not in vitals_dict["present_fields"]:
|
||||
vitals_dict["present_fields"].append("heart_rate")
|
||||
found = True
|
||||
|
||||
return found
|
||||
|
||||
|
||||
def _parse_288_byte_packet(data: bytes, vitals_dict: dict) -> bool:
|
||||
"""
|
||||
Parse 288-byte waveform packet (Subtype 21 for CMS8500).
|
||||
|
||||
Layout:
|
||||
[266-267] PR (LE u16) — pulse rate
|
||||
[268-269] SpO2% (LE u16) — live oxygen saturation percentage
|
||||
"""
|
||||
if len(data) < 270:
|
||||
return False
|
||||
|
||||
if "present_fields" not in vitals_dict:
|
||||
vitals_dict["present_fields"] = []
|
||||
if "spo2" not in vitals_dict["present_fields"]:
|
||||
vitals_dict["present_fields"].append("spo2")
|
||||
vitals_dict.setdefault("spo2", None)
|
||||
|
||||
found = False
|
||||
|
||||
# Offset 268: SpO2 percentage
|
||||
spo2_val = read_u16_le(data, 268)
|
||||
if spo2_val is not None and spo2_val != 65535 and spo2_val != 255 and 50 <= spo2_val <= 100:
|
||||
vitals_dict["spo2"] = float(spo2_val)
|
||||
found = True
|
||||
|
||||
# Offset 266: PR from SpO2 PR
|
||||
hr_val = read_u16_le(data, 266)
|
||||
if hr_val is not None and hr_val != 65535 and hr_val != 9999 and 20 <= hr_val <= 300:
|
||||
vitals_dict["heart_rate"] = float(hr_val)
|
||||
if "heart_rate" not in vitals_dict["present_fields"]:
|
||||
vitals_dict["present_fields"].append("heart_rate")
|
||||
found = True
|
||||
|
||||
return found
|
||||
|
||||
|
||||
def _parse_341_byte_packet(data: bytes, vitals_dict: dict) -> bool:
|
||||
"""
|
||||
Parse 341-byte waveform packet. Same tail layout as 286-byte.
|
||||
"""
|
||||
if len(data) < 270:
|
||||
return False
|
||||
|
||||
if "present_fields" not in vitals_dict:
|
||||
vitals_dict["present_fields"] = []
|
||||
if "spo2" not in vitals_dict["present_fields"]:
|
||||
vitals_dict["present_fields"].append("spo2")
|
||||
vitals_dict.setdefault("spo2", None)
|
||||
|
||||
found = False
|
||||
|
||||
spo2_val = read_u16_le(data, 264)
|
||||
if spo2_val is not None and spo2_val != 65535 and spo2_val != 255 and 50 <= spo2_val <= 100:
|
||||
vitals_dict["spo2"] = float(spo2_val)
|
||||
found = True
|
||||
|
||||
hr_val = read_u16_le(data, 266)
|
||||
if hr_val is not None and hr_val != 65535 and hr_val != 9999 and 20 <= hr_val <= 300:
|
||||
vitals_dict["heart_rate"] = float(hr_val)
|
||||
if "heart_rate" not in vitals_dict["present_fields"]:
|
||||
vitals_dict["present_fields"].append("heart_rate")
|
||||
found = True
|
||||
|
||||
return found
|
||||
|
||||
|
||||
def _parse_989_byte_packet(data: bytes, vitals_dict: dict) -> bool:
|
||||
"""
|
||||
Parse 989-byte waveform packet (Subtype 20).
|
||||
|
||||
Verified layout:
|
||||
[904-905] ECG Heart Rate (LE u16)
|
||||
65535 or 9999 = invalid/disconnected
|
||||
"""
|
||||
if len(data) < 906:
|
||||
return False
|
||||
|
||||
if "present_fields" not in vitals_dict:
|
||||
vitals_dict["present_fields"] = []
|
||||
if "heart_rate" not in vitals_dict["present_fields"]:
|
||||
vitals_dict["present_fields"].append("heart_rate")
|
||||
vitals_dict.setdefault("heart_rate", None)
|
||||
|
||||
found = False
|
||||
|
||||
hr_val = read_u16_le(data, 904)
|
||||
if hr_val is not None and hr_val != 65535 and hr_val != 9999 and 20 <= hr_val <= 300:
|
||||
vitals_dict["heart_rate"] = float(hr_val)
|
||||
found = True
|
||||
|
||||
return found
|
||||
|
||||
|
||||
def _find_vitals_block_in_stream(data: bytes, client_ip: str) -> Optional[NormalizedVitals]:
|
||||
"""
|
||||
Scan the data stream for Contec CMS7000PLUS sub-packets.
|
||||
Scan the data stream for Contec CMS7000PLUS sub-packets and parse them
|
||||
deterministically based on packet length.
|
||||
|
||||
Contec packets follow this framing pattern:
|
||||
[len_lo] [len_hi] [04] [46] [sub_type] [00] [sub_len_lo] [00] ...payload...
|
||||
|
||||
OR the data is a concatenation of multiple sub-packets separated by
|
||||
[xx][xx][04][47] (end marker?)
|
||||
|
||||
The key is to find [04][46] marker bytes and parse the packet.
|
||||
Packet types (verified by raw hex analysis):
|
||||
45 bytes (Subtype 22): NIBP Sys/Dia/MAP (u16 LE starting at offset 15)
|
||||
56 bytes (Subtype 23): Float32 Temperature (offset 8/12)
|
||||
286 bytes (Subtype 21): SpO2% at offset 264
|
||||
341 bytes (Subtype 21): SpO2% at offset 264
|
||||
989 bytes (Subtype 20): ECG Heart Rate at offset 904
|
||||
"""
|
||||
model = settings.device_models.get(client_ip, settings.monitor_model)
|
||||
vitals_dict = {
|
||||
"device_id": f"CMS7000PLUS_{client_ip}",
|
||||
"device_id": f"{model}_{client_ip}",
|
||||
"patient_id": "UNKNOWN",
|
||||
"timestamp": datetime.now(timezone.utc),
|
||||
"ip_address": client_ip,
|
||||
}
|
||||
found_any = False
|
||||
|
||||
# Scan through the buffer looking for [04 46] packet markers
|
||||
i = 0
|
||||
while i < len(data) - 8:
|
||||
# Look for the [04][46] marker which is the packet type indicator
|
||||
if data[i + 2] == 0x04 and data[i + 3] == 0x46:
|
||||
pkt_len = struct.unpack_from('<H', data, i)[0] # little-endian length
|
||||
if (data[i + 2] == 0x04 or data[i + 2] == 0x01 or data[i + 2] == 0x00) and data[i + 3] == 0x46:
|
||||
pkt_len = struct.unpack_from('<H', data, i)[0]
|
||||
|
||||
# Packet type B: small packets (< 600 bytes) with vitals summary
|
||||
# These have float32 NIBP values at offset +8 relative to packet start
|
||||
if 30 < pkt_len < 600 and i + 8 <= len(data):
|
||||
# Read float32 vitals at offsets 8, 12, 16, 20, 24 from packet start
|
||||
_parse_float_vitals(data, i + 8, vitals_dict)
|
||||
if 30 < pkt_len < 1050 and i + pkt_len <= len(data):
|
||||
pkt_data = data[i:i + pkt_len]
|
||||
|
||||
if pkt_len in (45, 47):
|
||||
_parse_45_byte_packet(pkt_data, vitals_dict)
|
||||
found_any = True
|
||||
elif pkt_len == 56:
|
||||
_parse_56_byte_packet(pkt_data, vitals_dict)
|
||||
found_any = True
|
||||
elif pkt_len == 286:
|
||||
_parse_286_byte_packet(pkt_data, vitals_dict)
|
||||
found_any = True
|
||||
elif pkt_len == 288:
|
||||
_parse_288_byte_packet(pkt_data, vitals_dict)
|
||||
found_any = True
|
||||
elif pkt_len == 341:
|
||||
_parse_341_byte_packet(pkt_data, vitals_dict)
|
||||
found_any = True
|
||||
elif pkt_len == 989:
|
||||
_parse_989_byte_packet(pkt_data, vitals_dict)
|
||||
found_any = True
|
||||
|
||||
# Find the vitals summary block in this packet
|
||||
# It appears near the tail of the resp waveform data
|
||||
# The pattern we look for: SpO2 byte (50-100), followed by various u16 values
|
||||
# In packet B, the summary is 75 bytes before the end
|
||||
if pkt_len > 50:
|
||||
# The summary block is consistently found around offset 322 in 397-byte packets
|
||||
# Relative to packet start: around pkt_len - 75
|
||||
summary_offset = i + max(4, pkt_len - 80)
|
||||
if summary_offset + 10 < len(data):
|
||||
# Scan in the last ~100 bytes of the packet for SpO2 pattern
|
||||
scan_end = min(i + pkt_len, len(data) - 4)
|
||||
scan_start = max(i + 4, scan_end - 100)
|
||||
for j in range(scan_start, scan_end - 4, 2):
|
||||
spo2 = read_u16_le(data, j)
|
||||
hr = read_u16_le(data, j + 2)
|
||||
pr = read_u16_le(data, j + 4)
|
||||
|
||||
# Check if this looks like [SpO2][HR/sentinel][PR]
|
||||
spo2_ok = spo2 is not None and 50 <= spo2 <= 100
|
||||
# HR can be 9999 (invalid/ECG disconnected) or valid 20-300
|
||||
hr_or_sentinel = hr is not None and (hr == INVALID_SENTINEL_U16 or 20 <= hr <= 300)
|
||||
pr_ok = pr is not None and (pr == INVALID_SENTINEL_U16 or 20 <= pr <= 300)
|
||||
|
||||
if spo2_ok and hr_or_sentinel and pr_ok:
|
||||
vitals_dict["spo2"] = float(spo2)
|
||||
found_any = True
|
||||
|
||||
# Use ECG HR if valid, else use SpO2 PR
|
||||
if hr is not None and hr != INVALID_SENTINEL_U16 and 20 <= hr <= 300:
|
||||
vitals_dict["heart_rate"] = float(hr)
|
||||
elif pr is not None and pr != INVALID_SENTINEL_U16 and 20 <= pr <= 300:
|
||||
vitals_dict["heart_rate"] = float(pr)
|
||||
|
||||
logger.info(
|
||||
f"Parsed vitals block at offset {j}: "
|
||||
f"SpO2={spo2}, HR={hr}, PR={pr}"
|
||||
)
|
||||
break
|
||||
|
||||
# Move past this packet
|
||||
i += max(pkt_len, 4)
|
||||
else:
|
||||
i += 1
|
||||
@@ -365,8 +535,10 @@ def parse_contec_data(raw_data: bytes, client_ip: str) -> Optional[NormalizedVit
|
||||
vitals = parse_contec_hl7_text(text, client_ip)
|
||||
if vitals:
|
||||
return vitals
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
logger.warning(f"HL7 message detected but failed to parse. Content: {repr(text)}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed decoding/parsing HL7: {e}")
|
||||
|
||||
# Attempt 2: Binary Parsing (Contec proprietary protocol)
|
||||
vitals = parse_contec_binary_packet(raw_data, client_ip)
|
||||
|
||||
Reference in New Issue
Block a user