Browse Source
Add device_model_catalog with sync-from-assets for pending series, enrich asset list with owner/agent fields and filters, preserve manual owners on report ingest, and record session handover (0.1.13-collector-ui). Co-authored-by: Cursor <cursoragent@cursor.com>master
35 changed files with 1926 additions and 77 deletions
@ -0,0 +1,3 @@
@@ -0,0 +1,3 @@
|
||||
# GeoNetAgent Collector |
||||
|
||||
FastAPI service for agent ingress and CMDB read API. |
||||
@ -0,0 +1,91 @@
@@ -0,0 +1,91 @@
|
||||
import uuid |
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, status |
||||
from sqlalchemy.orm import Session |
||||
|
||||
from app.core.auth import verify_api_bearer |
||||
from app.db.session import get_db |
||||
from app.schemas.agent_report import ErrorBody, ErrorResponse |
||||
from app.schemas.device_models import ( |
||||
DeviceModelCatalogIn, |
||||
DeviceModelCatalogOut, |
||||
DeviceModelCatalogPatch, |
||||
DeviceModelLookupOut, |
||||
DeviceModelSyncResult, |
||||
PaginatedDeviceModelCatalog, |
||||
) |
||||
from app.services import device_model_catalog_service as catalog_service |
||||
|
||||
router = APIRouter(prefix="/device-models") |
||||
|
||||
|
||||
def _bad_request(message: str) -> HTTPException: |
||||
return HTTPException( |
||||
status_code=status.HTTP_400_BAD_REQUEST, |
||||
detail=ErrorResponse(error=ErrorBody(code="BAD_REQUEST", message=message)).model_dump(), |
||||
) |
||||
|
||||
|
||||
@router.get("", response_model=PaginatedDeviceModelCatalog) |
||||
def list_device_models( |
||||
q: str | None = Query(None, max_length=200), |
||||
pending_only: bool = Query(False), |
||||
page: int = Query(1, ge=1), |
||||
limit: int = Query(50, ge=1, le=200), |
||||
db: Session = Depends(get_db), |
||||
_: str = Depends(verify_api_bearer), |
||||
) -> PaginatedDeviceModelCatalog: |
||||
return catalog_service.list_catalog(db, q=q, pending_only=pending_only, page=page, limit=limit) |
||||
|
||||
|
||||
@router.post("/sync-from-assets", response_model=DeviceModelSyncResult) |
||||
def sync_device_models_from_assets( |
||||
db: Session = Depends(get_db), |
||||
_: str = Depends(verify_api_bearer), |
||||
) -> DeviceModelSyncResult: |
||||
return catalog_service.sync_missing_from_assets(db) |
||||
|
||||
|
||||
@router.get("/lookup", response_model=DeviceModelLookupOut) |
||||
def lookup_device_model( |
||||
manufacturer: str = Query(..., min_length=1, max_length=100), |
||||
model: str = Query(..., min_length=1, max_length=100), |
||||
db: Session = Depends(get_db), |
||||
_: str = Depends(verify_api_bearer), |
||||
) -> DeviceModelLookupOut: |
||||
return catalog_service.resolve_series(db, manufacturer, model) |
||||
|
||||
|
||||
@router.post("", response_model=DeviceModelCatalogOut, status_code=status.HTTP_201_CREATED) |
||||
def create_device_model( |
||||
body: DeviceModelCatalogIn, |
||||
db: Session = Depends(get_db), |
||||
_: str = Depends(verify_api_bearer), |
||||
) -> DeviceModelCatalogOut: |
||||
try: |
||||
return catalog_service.create_catalog_entry(db, body) |
||||
except ValueError as exc: |
||||
raise _bad_request(str(exc)) from exc |
||||
|
||||
|
||||
@router.patch("/{entry_id}", response_model=DeviceModelCatalogOut) |
||||
def patch_device_model( |
||||
entry_id: uuid.UUID, |
||||
body: DeviceModelCatalogPatch, |
||||
db: Session = Depends(get_db), |
||||
_: str = Depends(verify_api_bearer), |
||||
) -> DeviceModelCatalogOut: |
||||
result = catalog_service.patch_catalog_entry(db, entry_id, body) |
||||
if result is None: |
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Catalog entry not found") |
||||
return result |
||||
|
||||
|
||||
@router.delete("/{entry_id}", status_code=status.HTTP_204_NO_CONTENT) |
||||
def delete_device_model( |
||||
entry_id: uuid.UUID, |
||||
db: Session = Depends(get_db), |
||||
_: str = Depends(verify_api_bearer), |
||||
) -> None: |
||||
if not catalog_service.delete_catalog_entry(db, entry_id): |
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Catalog entry not found") |
||||
@ -1,9 +1,10 @@
@@ -1,9 +1,10 @@
|
||||
from fastapi import APIRouter |
||||
|
||||
from app.api.v1 import agent, assets, health, ldap |
||||
from app.api.v1 import agent, assets, device_models, health, ldap |
||||
|
||||
router = APIRouter(prefix="/api/v1") |
||||
router.include_router(health.router, tags=["health"]) |
||||
router.include_router(agent.router, tags=["agent"]) |
||||
router.include_router(assets.router, tags=["assets"]) |
||||
router.include_router(device_models.router, tags=["device-models"]) |
||||
router.include_router(ldap.router, tags=["ldap"]) |
||||
|
||||
@ -0,0 +1,52 @@
@@ -0,0 +1,52 @@
|
||||
from datetime import datetime |
||||
|
||||
from pydantic import BaseModel, Field |
||||
|
||||
|
||||
class DeviceModelCatalogIn(BaseModel): |
||||
manufacturer: str = Field(..., min_length=1, max_length=100) |
||||
model: str = Field(..., min_length=1, max_length=100) |
||||
series: str = Field(..., min_length=1, max_length=255) |
||||
source: str = Field(default="manual", max_length=50) |
||||
notes: str | None = Field(default=None, max_length=2000) |
||||
|
||||
|
||||
class DeviceModelCatalogPatch(BaseModel): |
||||
series: str | None = Field(default=None, min_length=1, max_length=255) |
||||
source: str | None = Field(default=None, max_length=50) |
||||
notes: str | None = Field(default=None, max_length=2000) |
||||
|
||||
|
||||
class DeviceModelCatalogOut(BaseModel): |
||||
id: str |
||||
manufacturer: str |
||||
model: str |
||||
series: str | None = None |
||||
series_pending: bool = False |
||||
source: str |
||||
notes: str | None = None |
||||
created_at: datetime |
||||
updated_at: datetime |
||||
|
||||
|
||||
class DeviceModelSyncResult(BaseModel): |
||||
inserted: int |
||||
already_exists: int |
||||
pending_series: int |
||||
distinct_asset_models: int |
||||
|
||||
|
||||
class PaginatedDeviceModelCatalog(BaseModel): |
||||
items: list[DeviceModelCatalogOut] |
||||
total: int |
||||
page: int |
||||
limit: int |
||||
pages: int |
||||
|
||||
|
||||
class DeviceModelLookupOut(BaseModel): |
||||
manufacturer: str |
||||
model: str |
||||
series: str | None = None |
||||
catalog_id: str | None = None |
||||
matched: bool = False |
||||
@ -0,0 +1,321 @@
@@ -0,0 +1,321 @@
|
||||
import math |
||||
import re |
||||
import uuid |
||||
|
||||
from sqlalchemy import case, func, or_, select |
||||
from sqlalchemy.orm import Session |
||||
|
||||
from app.db.models import Asset, DeviceModelCatalog |
||||
from app.schemas.device_models import ( |
||||
DeviceModelCatalogIn, |
||||
DeviceModelCatalogOut, |
||||
DeviceModelCatalogPatch, |
||||
DeviceModelLookupOut, |
||||
DeviceModelSyncResult, |
||||
PaginatedDeviceModelCatalog, |
||||
) |
||||
|
||||
_MANUFACTURER_SUFFIXES = re.compile(r"\b(INC\.?|LTD\.?|CO\.?|CORP\.?|GMBH|S\.A\.?)\b", re.I) |
||||
|
||||
|
||||
def normalize_manufacturer(value: str | None) -> str | None: |
||||
if not value: |
||||
return None |
||||
text = re.sub(r"\s+", " ", value.strip().upper()) |
||||
text = _MANUFACTURER_SUFFIXES.sub("", text) |
||||
text = re.sub(r"[^\w\s&.-]", "", text) |
||||
text = re.sub(r"\s+", " ", text).strip() |
||||
return text or None |
||||
|
||||
|
||||
def normalize_model(value: str | None) -> str | None: |
||||
if not value: |
||||
return None |
||||
text = re.sub(r"\s+", " ", value.strip().upper()) |
||||
return text or None |
||||
|
||||
|
||||
def catalog_key(manufacturer: str | None, model: str | None) -> tuple[str, str] | None: |
||||
mfr = normalize_manufacturer(manufacturer) |
||||
mdl = normalize_model(model) |
||||
if not mfr or not mdl: |
||||
return None |
||||
return mfr, mdl |
||||
|
||||
|
||||
def _series_pending(series: str | None) -> bool: |
||||
return not (series or "").strip() |
||||
|
||||
|
||||
def _to_out(row: DeviceModelCatalog) -> DeviceModelCatalogOut: |
||||
pending = _series_pending(row.series) |
||||
return DeviceModelCatalogOut( |
||||
id=str(row.id), |
||||
manufacturer=row.manufacturer, |
||||
model=row.model, |
||||
series=None if pending else (row.series or "").strip(), |
||||
series_pending=pending, |
||||
source=row.source, |
||||
notes=row.notes, |
||||
created_at=row.created_at, |
||||
updated_at=row.updated_at, |
||||
) |
||||
|
||||
|
||||
def resolve_series( |
||||
db: Session, |
||||
manufacturer: str | None, |
||||
model: str | None, |
||||
) -> DeviceModelLookupOut: |
||||
key = catalog_key(manufacturer, model) |
||||
if not key: |
||||
return DeviceModelLookupOut( |
||||
manufacturer=manufacturer or "", |
||||
model=model or "", |
||||
series=None, |
||||
matched=False, |
||||
) |
||||
|
||||
row = db.scalar( |
||||
select(DeviceModelCatalog).where( |
||||
DeviceModelCatalog.manufacturer_normalized == key[0], |
||||
DeviceModelCatalog.model_normalized == key[1], |
||||
) |
||||
) |
||||
if not row or _series_pending(row.series): |
||||
return DeviceModelLookupOut( |
||||
manufacturer=manufacturer or "", |
||||
model=model or "", |
||||
series=None, |
||||
catalog_id=str(row.id) if row else None, |
||||
matched=False, |
||||
) |
||||
|
||||
return DeviceModelLookupOut( |
||||
manufacturer=row.manufacturer, |
||||
model=row.model, |
||||
series=(row.series or "").strip(), |
||||
catalog_id=str(row.id), |
||||
matched=True, |
||||
) |
||||
|
||||
|
||||
def resolve_series_map( |
||||
db: Session, |
||||
pairs: list[tuple[str | None, str | None]], |
||||
) -> dict[tuple[str, str], str]: |
||||
keys: set[tuple[str, str]] = set() |
||||
for manufacturer, model in pairs: |
||||
key = catalog_key(manufacturer, model) |
||||
if key: |
||||
keys.add(key) |
||||
if not keys: |
||||
return {} |
||||
|
||||
mfrs = {k[0] for k in keys} |
||||
mdls = {k[1] for k in keys} |
||||
rows = db.scalars( |
||||
select(DeviceModelCatalog).where( |
||||
DeviceModelCatalog.manufacturer_normalized.in_(mfrs), |
||||
DeviceModelCatalog.model_normalized.in_(mdls), |
||||
) |
||||
).all() |
||||
|
||||
result: dict[tuple[str, str], str] = {} |
||||
for row in rows: |
||||
key = (row.manufacturer_normalized, row.model_normalized) |
||||
if key in keys and not _series_pending(row.series): |
||||
result[key] = (row.series or "").strip() |
||||
return result |
||||
|
||||
|
||||
def list_catalog( |
||||
db: Session, |
||||
*, |
||||
q: str | None = None, |
||||
pending_only: bool = False, |
||||
page: int = 1, |
||||
limit: int = 50, |
||||
) -> PaginatedDeviceModelCatalog: |
||||
stmt = select(DeviceModelCatalog) |
||||
if pending_only: |
||||
stmt = stmt.where( |
||||
or_( |
||||
DeviceModelCatalog.series.is_(None), |
||||
DeviceModelCatalog.series == "", |
||||
) |
||||
) |
||||
if q: |
||||
pattern = f"%{q.strip()}%" |
||||
stmt = stmt.where( |
||||
or_( |
||||
DeviceModelCatalog.manufacturer.ilike(pattern), |
||||
DeviceModelCatalog.model.ilike(pattern), |
||||
DeviceModelCatalog.series.ilike(pattern), |
||||
DeviceModelCatalog.manufacturer_normalized.ilike(pattern), |
||||
DeviceModelCatalog.model_normalized.ilike(pattern), |
||||
) |
||||
) |
||||
|
||||
total = db.scalar(select(func.count()).select_from(stmt.subquery())) or 0 |
||||
pages = max(1, math.ceil(total / limit)) if total else 0 |
||||
offset = (page - 1) * limit |
||||
rows = db.scalars( |
||||
stmt.order_by( |
||||
case( |
||||
(or_(DeviceModelCatalog.series.is_(None), DeviceModelCatalog.series == ""), 0), |
||||
else_=1, |
||||
), |
||||
DeviceModelCatalog.manufacturer_normalized, |
||||
DeviceModelCatalog.model_normalized, |
||||
) |
||||
.offset(offset) |
||||
.limit(limit) |
||||
).all() |
||||
|
||||
return PaginatedDeviceModelCatalog( |
||||
items=[_to_out(row) for row in rows], |
||||
total=total, |
||||
page=page, |
||||
limit=limit, |
||||
pages=pages, |
||||
) |
||||
|
||||
|
||||
def create_catalog_entry(db: Session, data: DeviceModelCatalogIn) -> DeviceModelCatalogOut: |
||||
key = catalog_key(data.manufacturer, data.model) |
||||
if not key: |
||||
raise ValueError("manufacturer and model are required") |
||||
|
||||
existing = db.scalar( |
||||
select(DeviceModelCatalog).where( |
||||
DeviceModelCatalog.manufacturer_normalized == key[0], |
||||
DeviceModelCatalog.model_normalized == key[1], |
||||
) |
||||
) |
||||
if existing: |
||||
existing.series = data.series.strip() |
||||
existing.source = (data.source or "manual").strip() or "manual" |
||||
existing.notes = data.notes |
||||
existing.manufacturer = data.manufacturer.strip() |
||||
existing.model = data.model.strip() |
||||
db.commit() |
||||
db.refresh(existing) |
||||
return _to_out(existing) |
||||
|
||||
row = DeviceModelCatalog( |
||||
manufacturer=data.manufacturer.strip(), |
||||
model=data.model.strip(), |
||||
manufacturer_normalized=key[0], |
||||
model_normalized=key[1], |
||||
series=data.series.strip(), |
||||
source=(data.source or "manual").strip() or "manual", |
||||
notes=data.notes, |
||||
) |
||||
db.add(row) |
||||
db.commit() |
||||
db.refresh(row) |
||||
return _to_out(row) |
||||
|
||||
|
||||
def patch_catalog_entry( |
||||
db: Session, |
||||
entry_id: uuid.UUID, |
||||
data: DeviceModelCatalogPatch, |
||||
) -> DeviceModelCatalogOut | None: |
||||
row = db.scalar(select(DeviceModelCatalog).where(DeviceModelCatalog.id == entry_id)) |
||||
if not row: |
||||
return None |
||||
|
||||
if data.series is not None: |
||||
row.series = data.series.strip() |
||||
if data.source is not None: |
||||
row.source = data.source.strip() or "manual" |
||||
if data.notes is not None: |
||||
row.notes = data.notes or None |
||||
|
||||
db.commit() |
||||
db.refresh(row) |
||||
return _to_out(row) |
||||
|
||||
|
||||
def delete_catalog_entry(db: Session, entry_id: uuid.UUID) -> bool: |
||||
row = db.scalar(select(DeviceModelCatalog).where(DeviceModelCatalog.id == entry_id)) |
||||
if not row: |
||||
return False |
||||
db.delete(row) |
||||
db.commit() |
||||
return True |
||||
|
||||
|
||||
def sync_missing_from_assets(db: Session) -> DeviceModelSyncResult: |
||||
"""Insert distinct asset manufacturer+model pairs missing from catalog (series kosong).""" |
||||
pairs = db.execute( |
||||
select(Asset.manufacturer, Asset.model) |
||||
.where( |
||||
Asset.deleted_at.is_(None), |
||||
Asset.manufacturer.isnot(None), |
||||
Asset.model.isnot(None), |
||||
Asset.manufacturer != "", |
||||
Asset.model != "", |
||||
) |
||||
.distinct() |
||||
).all() |
||||
|
||||
inserted = 0 |
||||
already_exists = 0 |
||||
pending_series = 0 |
||||
|
||||
for manufacturer, model in pairs: |
||||
key = catalog_key(manufacturer, model) |
||||
if not key: |
||||
continue |
||||
|
||||
existing = db.scalar( |
||||
select(DeviceModelCatalog).where( |
||||
DeviceModelCatalog.manufacturer_normalized == key[0], |
||||
DeviceModelCatalog.model_normalized == key[1], |
||||
) |
||||
) |
||||
if existing: |
||||
already_exists += 1 |
||||
if _series_pending(existing.series): |
||||
pending_series += 1 |
||||
continue |
||||
|
||||
db.add( |
||||
DeviceModelCatalog( |
||||
manufacturer=manufacturer.strip(), |
||||
model=model.strip(), |
||||
manufacturer_normalized=key[0], |
||||
model_normalized=key[1], |
||||
series=None, |
||||
source="asset-discovered", |
||||
notes="Auto dari assets — isi series", |
||||
) |
||||
) |
||||
inserted += 1 |
||||
pending_series += 1 |
||||
|
||||
if inserted: |
||||
db.commit() |
||||
else: |
||||
db.rollback() |
||||
|
||||
pending_total = db.scalar( |
||||
select(func.count()) |
||||
.select_from(DeviceModelCatalog) |
||||
.where( |
||||
or_( |
||||
DeviceModelCatalog.series.is_(None), |
||||
DeviceModelCatalog.series == "", |
||||
) |
||||
) |
||||
) or 0 |
||||
|
||||
return DeviceModelSyncResult( |
||||
inserted=inserted, |
||||
already_exists=already_exists, |
||||
pending_series=pending_total, |
||||
distinct_asset_models=len(pairs), |
||||
) |
||||
@ -0,0 +1,37 @@
@@ -0,0 +1,37 @@
|
||||
#!/bin/bash |
||||
set -euo pipefail |
||||
TOKEN=$(grep -E '^AGENT_BEARER_TOKEN=' /opt/stacks/geonetagent-collector/.env | cut -d= -f2- | tr -d '\r"') |
||||
echo "=== stats ===" |
||||
curl -sf -H "Authorization: Bearer $TOKEN" http://127.0.0.1:18000/api/v1/health |
||||
echo |
||||
curl -sf -H "Authorization: Bearer $TOKEN" http://127.0.0.1:18000/api/v1/stats/summary |
||||
echo |
||||
echo "=== assets summary ===" |
||||
curl -sf -H "Authorization: Bearer $TOKEN" 'http://127.0.0.1:18000/api/v1/assets?limit=50' | python3 <<'PY' |
||||
import json,sys |
||||
from datetime import datetime, timezone, timedelta |
||||
d=json.load(sys.stdin) |
||||
now=datetime.now(timezone.utc) |
||||
cut=now-timedelta(hours=24) |
||||
fresh=stale=none=0 |
||||
for i in d['items']: |
||||
ls=i.get('last_seen_at') |
||||
if not ls: |
||||
none+=1 |
||||
continue |
||||
t=datetime.fromisoformat(ls.replace('Z','+00:00')) |
||||
if t>=cut: fresh+=1 |
||||
else: stale+=1 |
||||
print(f"page_items={len(d['items'])} total_in_db={d['total']}") |
||||
print(f"fresh_24h={fresh} stale_24h={stale} no_last_seen={none} (on this page)") |
||||
print("--- newest ---") |
||||
for i in sorted(d['items'], key=lambda x: x.get('last_seen_at') or '', reverse=True)[:10]: |
||||
print(i['hostname'], i.get('last_seen_at'), i.get('agent_version')) |
||||
print("--- oldest on page ---") |
||||
for i in sorted(d['items'], key=lambda x: x.get('last_seen_at') or '')[:5]: |
||||
print(i['hostname'], i.get('last_seen_at'), i.get('agent_version')) |
||||
PY |
||||
echo "=== search DESKTOP-RRJ9G01 ===" |
||||
curl -sf -H "Authorization: Bearer $TOKEN" 'http://127.0.0.1:18000/api/v1/assets?q=DESKTOP-RRJ9G01&limit=5' | python3 -m json.tool |
||||
echo "=== collector logs (agent) ===" |
||||
docker logs geonet-collector --tail 40 2>&1 | grep -iE 'report|401|403|error|DESKTOP' || docker logs geonet-collector --tail 15 |
||||
@ -0,0 +1,17 @@
@@ -0,0 +1,17 @@
|
||||
SELECT a.hostname, a.last_seen_at, u.username, aua.is_primary, aua.assigned_by |
||||
FROM assets a |
||||
LEFT JOIN asset_user_assignments aua ON aua.asset_id = a.id AND aua.ended_at IS NULL |
||||
LEFT JOIN users u ON u.id = aua.user_id |
||||
WHERE a.deleted_at IS NULL |
||||
ORDER BY a.last_seen_at DESC NULLS LAST |
||||
LIMIT 20; |
||||
|
||||
SELECT a.hostname, u.username, count(*) |
||||
FROM asset_user_assignments aua |
||||
JOIN assets a ON a.id = aua.asset_id |
||||
JOIN users u ON u.id = aua.user_id |
||||
WHERE aua.is_primary = true AND aua.ended_at IS NULL |
||||
GROUP BY a.hostname, u.username |
||||
HAVING count(*) > 1; |
||||
|
||||
SELECT hostname, last_seen_at FROM assets WHERE hostname ILIKE '%RRJ9G01%'; |
||||
@ -0,0 +1,9 @@
@@ -0,0 +1,9 @@
|
||||
SELECT id, hostname, machine_id, smbios_uuid, serial_number, primary_physical_mac |
||||
FROM assets |
||||
WHERE hostname ILIKE '%RRJ9G01%' AND deleted_at IS NULL; |
||||
|
||||
SELECT u.username, aua.* |
||||
FROM asset_user_assignments aua |
||||
JOIN users u ON u.id = aua.user_id |
||||
JOIN assets a ON a.id = aua.asset_id |
||||
WHERE a.hostname = 'DESKTOP-RRJ9G01'; |
||||
@ -0,0 +1,31 @@
@@ -0,0 +1,31 @@
|
||||
import json |
||||
import traceback |
||||
import uuid |
||||
from datetime import datetime, timezone |
||||
|
||||
from app.db.session import SessionLocal |
||||
from app.schemas.agent_report import AgentReportRequest, SystemInfo, HealthInfo |
||||
from app.services.asset_service import ingest_agent_report |
||||
|
||||
report = AgentReportRequest( |
||||
schema_version="1.0", |
||||
agent_version="0.1.12-phase0", |
||||
collected_at=datetime.now(timezone.utc), |
||||
hostname="DESKTOP-RRJ9G01", |
||||
machine_id="test-machine-id", |
||||
smbios_uuid="00000000-0000-0000-0000-000000000001", |
||||
serial_number="TESTSERIAL", |
||||
asset_type="endpoint", |
||||
logged_in_user="DESKTOP-RRJ9G01\\admin", |
||||
system=SystemInfo(manufacturer="LENOVO", model="82AU", os_name="Windows 11"), |
||||
health=HealthInfo(), |
||||
software=[{"name": "Test App", "version": "1.0"}], |
||||
) |
||||
|
||||
db = SessionLocal() |
||||
try: |
||||
result = ingest_agent_report(db, report) |
||||
print("OK", result.model_dump()) |
||||
except Exception: |
||||
db.rollback() |
||||
traceback.print_exc() |
||||
@ -0,0 +1,24 @@
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env python3 |
||||
import json |
||||
import sys |
||||
import traceback |
||||
|
||||
from app.db.session import SessionLocal |
||||
from app.schemas.agent_report import AgentReportRequest |
||||
from app.services.asset_service import ingest_agent_report |
||||
|
||||
path = sys.argv[1] if len(sys.argv) > 1 else "/tmp/geonet-report-test.json" |
||||
with open(path, encoding="utf-8") as f: |
||||
data = json.load(f) |
||||
|
||||
report = AgentReportRequest.model_validate(data) |
||||
db = SessionLocal() |
||||
try: |
||||
result = ingest_agent_report(db, report) |
||||
print("OK", result.model_dump()) |
||||
except Exception: |
||||
db.rollback() |
||||
traceback.print_exc() |
||||
sys.exit(1) |
||||
finally: |
||||
db.close() |
||||
@ -0,0 +1,32 @@
@@ -0,0 +1,32 @@
|
||||
import traceback |
||||
from datetime import datetime, timezone |
||||
|
||||
from app.db.session import SessionLocal |
||||
from app.schemas.agent_report import AgentReportRequest, SystemInfo, HealthInfo, AssetIdentity |
||||
from app.services.asset_service import ingest_agent_report |
||||
|
||||
report = AgentReportRequest( |
||||
schema_version="1.0", |
||||
agent_version="0.1.12-phase0", |
||||
collected_at=datetime.now(timezone.utc), |
||||
hostname="DESKTOP-RRJ9G01", |
||||
machine_id="3709baa5-5265-4e4d-9a81-fb8c8927bb3b", |
||||
smbios_uuid="03D502E0-045E-0550-8E06-BA0700080009", |
||||
asset_type="endpoint", |
||||
logged_in_user="DESKTOP-RRJ9G01\\admin", |
||||
asset_identity=AssetIdentity(primary_physical_mac="E0:D5:5E:50:8E:BA"), |
||||
system=SystemInfo(manufacturer="LENOVO", model="82AU", os_name="Windows 11"), |
||||
health=HealthInfo(), |
||||
software=[{"name": "Test App", "version": "1.0"}], |
||||
) |
||||
|
||||
db = SessionLocal() |
||||
try: |
||||
result = ingest_agent_report(db, report) |
||||
print("OK", result.model_dump()) |
||||
except Exception: |
||||
db.rollback() |
||||
traceback.print_exc() |
||||
raise |
||||
finally: |
||||
db.close() |
||||
@ -0,0 +1,19 @@
@@ -0,0 +1,19 @@
|
||||
import urllib.parse |
||||
import urllib.request |
||||
|
||||
from app.config import get_settings |
||||
|
||||
token = get_settings().resolved_api_bearer_token() |
||||
headers = {"Authorization": f"Bearer {token}"} |
||||
|
||||
|
||||
def get(url: str) -> str: |
||||
req = urllib.request.Request(url, headers=headers) |
||||
with urllib.request.urlopen(req) as resp: |
||||
return resp.read().decode() |
||||
|
||||
|
||||
base = "http://127.0.0.1:8000" |
||||
print(get(f"{base}/api/v1/device-models?limit=3")) |
||||
q = urllib.parse.urlencode({"manufacturer": "LENOVO", "model": "82AU"}) |
||||
print(get(f"{base}/api/v1/device-models/lookup?{q}")) |
||||
@ -0,0 +1,10 @@
@@ -0,0 +1,10 @@
|
||||
#!/bin/bash |
||||
set -euo pipefail |
||||
ENV_FILE="${1:-/opt/stacks/geonetagent-collector/.env}" |
||||
TOKEN=$(grep -E '^AGENT_BEARER_TOKEN=' "$ENV_FILE" | cut -d= -f2- | tr -d '\r"') |
||||
curl -sf -H "Authorization: Bearer $TOKEN" 'http://127.0.0.1:18000/api/v1/assets?limit=2' | python3 -c " |
||||
import json,sys |
||||
d=json.load(sys.stdin) |
||||
for i in d.get('items',[])[:2]: |
||||
print(f\"{i.get('hostname')}: owner={i.get('owner_display_name')} / {i.get('owner_ldap_username')} agent={i.get('agent_version')}\") |
||||
" |
||||
@ -0,0 +1,9 @@
@@ -0,0 +1,9 @@
|
||||
#!/bin/bash |
||||
set -euo pipefail |
||||
TOKEN=$(grep -E '^AGENT_BEARER_TOKEN=' /opt/stacks/geonetagent-collector/.env | cut -d= -f2- | tr -d '\r"') |
||||
echo "=== owner search ===" |
||||
curl -sf -H "Authorization: Bearer $TOKEN" 'http://127.0.0.1:18000/api/v1/assets?owner=rbsetiawan&limit=3' |
||||
echo |
||||
echo "=== series sort ===" |
||||
curl -sf -H "Authorization: Bearer $TOKEN" 'http://127.0.0.1:18000/api/v1/assets?sort=series&order=asc&limit=2' |
||||
echo |
||||
@ -0,0 +1,13 @@
@@ -0,0 +1,13 @@
|
||||
#!/bin/bash |
||||
set -euo pipefail |
||||
ENV_FILE="${1:-/opt/stacks/geonetagent-collector/.env}" |
||||
TOKEN=$(grep -E '^AGENT_BEARER_TOKEN=' "$ENV_FILE" | cut -d= -f2- | tr -d '\r"') |
||||
echo "token_len=${#TOKEN}" |
||||
echo "=== health ===" |
||||
curl -s -w "\nHTTP:%{http_code}\n" -H "Authorization: Bearer $TOKEN" http://127.0.0.1:18000/api/v1/health |
||||
echo "=== sync-from-assets ===" |
||||
curl -s -w "\nHTTP:%{http_code}\n" -X POST -H "Authorization: Bearer $TOKEN" \ |
||||
http://127.0.0.1:18000/api/v1/device-models/sync-from-assets |
||||
echo "=== pending (limit 3) ===" |
||||
curl -s -w "\nHTTP:%{http_code}\n" -H "Authorization: Bearer $TOKEN" \ |
||||
'http://127.0.0.1:18000/api/v1/device-models?pending_only=true&limit=3' |
||||
@ -0,0 +1,138 @@
@@ -0,0 +1,138 @@
|
||||
from app.services.device_model_catalog_service import ( |
||||
catalog_key, |
||||
normalize_manufacturer, |
||||
normalize_model, |
||||
) |
||||
|
||||
|
||||
def test_normalize_manufacturer_strips_suffix(): |
||||
assert normalize_manufacturer("Dell Inc.") == "DELL" |
||||
assert normalize_manufacturer("LENOVO") == "LENOVO" |
||||
|
||||
|
||||
def test_normalize_model_uppercase(): |
||||
assert normalize_model("82au") == "82AU" |
||||
assert normalize_model(" Inspiron 7447 ") == "INSPIRON 7447" |
||||
|
||||
|
||||
def test_catalog_key_requires_both(): |
||||
assert catalog_key("LENOVO", "82AU") == ("LENOVO", "82AU") |
||||
assert catalog_key(None, "82AU") is None |
||||
assert catalog_key("LENOVO", "") is None |
||||
|
||||
|
||||
def test_create_catalog_upsert(monkeypatch): |
||||
from sqlalchemy import create_engine |
||||
from sqlalchemy.orm import sessionmaker |
||||
from sqlalchemy.pool import StaticPool |
||||
|
||||
from app.db.models import DeviceModelCatalog |
||||
from app.db.session import Base |
||||
from app.schemas.device_models import DeviceModelCatalogIn |
||||
from app.services import device_model_catalog_service as svc |
||||
|
||||
engine = create_engine( |
||||
"sqlite://", |
||||
connect_args={"check_same_thread": False}, |
||||
poolclass=StaticPool, |
||||
) |
||||
Base.metadata.create_all(engine, tables=[DeviceModelCatalog.__table__]) |
||||
Session = sessionmaker(bind=engine) |
||||
db = Session() |
||||
|
||||
created = svc.create_catalog_entry( |
||||
db, |
||||
DeviceModelCatalogIn( |
||||
manufacturer="LENOVO", |
||||
model="82AU", |
||||
series="Lenovo Legion 5-15IMH05", |
||||
notes="test", |
||||
), |
||||
) |
||||
assert created.series == "Lenovo Legion 5-15IMH05" |
||||
|
||||
updated = svc.create_catalog_entry( |
||||
db, |
||||
DeviceModelCatalogIn( |
||||
manufacturer="Lenovo", |
||||
model="82au", |
||||
series="Lenovo Legion 5-15IMH05H", |
||||
), |
||||
) |
||||
assert updated.id == created.id |
||||
assert updated.series == "Lenovo Legion 5-15IMH05H" |
||||
|
||||
lookup = svc.resolve_series(db, "LENOVO", "82AU") |
||||
assert lookup.matched is True |
||||
assert lookup.series == "Lenovo Legion 5-15IMH05H" |
||||
|
||||
series_map = svc.resolve_series_map(db, [("LENOVO", "82AU"), ("Dell Inc.", "XPS")]) |
||||
assert series_map[("LENOVO", "82AU")] == "Lenovo Legion 5-15IMH05H" |
||||
assert ("DELL", "XPS") not in series_map |
||||
|
||||
db.close() |
||||
|
||||
|
||||
def test_sync_missing_from_assets_inserts_pending(): |
||||
from sqlalchemy import create_engine, select |
||||
from sqlalchemy.orm import sessionmaker |
||||
from sqlalchemy.pool import StaticPool |
||||
|
||||
from app.db.models import Asset, DeviceModelCatalog |
||||
from app.db.session import Base |
||||
from app.schemas.device_models import DeviceModelCatalogPatch |
||||
from app.services import device_model_catalog_service as svc |
||||
|
||||
engine = create_engine( |
||||
"sqlite://", |
||||
connect_args={"check_same_thread": False}, |
||||
poolclass=StaticPool, |
||||
) |
||||
Base.metadata.create_all(engine, tables=[DeviceModelCatalog.__table__, Asset.__table__]) |
||||
Session = sessionmaker(bind=engine) |
||||
db = Session() |
||||
|
||||
db.add( |
||||
Asset( |
||||
asset_type="endpoint", |
||||
hostname="pc1", |
||||
manufacturer="Dell Inc.", |
||||
model="0X8DX", |
||||
) |
||||
) |
||||
db.add( |
||||
Asset( |
||||
asset_type="endpoint", |
||||
hostname="pc2", |
||||
manufacturer="Dell Inc.", |
||||
model="0X8DX", |
||||
) |
||||
) |
||||
db.commit() |
||||
|
||||
result = svc.sync_missing_from_assets(db) |
||||
assert result.inserted == 1 |
||||
assert result.pending_series == 1 |
||||
assert result.distinct_asset_models == 1 |
||||
|
||||
row = db.scalar( |
||||
select(DeviceModelCatalog).where(DeviceModelCatalog.model_normalized == "0X8DX") |
||||
) |
||||
assert row is not None |
||||
assert row.series is None |
||||
assert row.source == "asset-discovered" |
||||
|
||||
lookup = svc.resolve_series(db, "Dell Inc.", "0X8DX") |
||||
assert lookup.matched is False |
||||
assert lookup.catalog_id == str(row.id) |
||||
|
||||
svc.patch_catalog_entry(db, row.id, DeviceModelCatalogPatch(series="OptiPlex 7090")) |
||||
lookup2 = svc.resolve_series(db, "DELL", "0X8DX") |
||||
assert lookup2.matched is True |
||||
assert lookup2.series == "OptiPlex 7090" |
||||
|
||||
result2 = svc.sync_missing_from_assets(db) |
||||
assert result2.inserted == 0 |
||||
assert result2.already_exists == 1 |
||||
|
||||
db.close() |
||||
@ -1,25 +1,39 @@
@@ -1,25 +1,39 @@
|
||||
# AI Session Summary |
||||
|
||||
```yaml |
||||
session: 2026-07-06-docs-bootstrap |
||||
session: 2026-07-08-cmdb-catalog-ui |
||||
project: GeoNetAgent/ |
||||
version: 0.1.12-phase0 |
||||
version: 0.1.13-collector-ui |
||||
status: ready |
||||
next: Lanjutkan fase aktif di .cursor/progress.md |
||||
next: Rollout agent ke endpoint stale; backfill owner display_name via LDAP re-select |
||||
agent: Cursor |
||||
``` |
||||
|
||||
## TL;DR |
||||
|
||||
- Bootstrap docs AI-Agent-Standards: version.md, system-requirements.md, guide/, ai-session-summary |
||||
- Agent live `v0.1.12-phase0`; collector di `10.100.1.24` |
||||
- Device model catalog SSOT (`device_model_catalog`) + sync dari assets (series pending) — live UI |
||||
- CMDB UI: Owner (nama+username), Agent version, search/filter/sort (owner/model/series) |
||||
- Bug ingest 500: agent vs owner manual → unique primary — **fixed** (manual owner tidak ditimpa) |
||||
- Deploy: migrasi `006`/`007` di `.25`; collector rebuild di `.24` (`geonet-collector`) |
||||
|
||||
## Lanjut dari sini |
||||
|
||||
1. Baca `.cursor/progress.md` untuk fase aktif |
||||
2. Bump versi agent → append `docs/version.md` + publish |
||||
1. UI: [collector.gisportal.id/ui/](https://collector.gisportal.id/ui/) — isi Series pending; filter stale + rollout install agent |
||||
2. Backfill `users.display_name`: di Edit Asset, pilih ulang owner dari LDAP autocomplete → Simpan |
||||
3. Opsional: hapus asset uji `DESKTOP-RRJ9G01-00000000` (TESTSERIAL dari ingest minimal) |
||||
|
||||
## Jangan |
||||
|
||||
- Jangan hardcode credential di skrip PowerShell |
||||
- Jangan commit `.env` |
||||
- Jangan commit `agent/config.json` / `.env` (token) |
||||
- Jangan biarkan agent overwrite owner `assigned_by=manual` |
||||
- Jangan hardcode URL collector di kode — pakai config/`AGENT_*` env |
||||
|
||||
## Detail (deploy) |
||||
|
||||
| Host | Aksi | |
||||
|------|------| |
||||
| `10.100.1.25` | `006_device_model_catalog.sql`, `007_..._pending_series.sql` applied | |
||||
| `10.100.1.24` | `/opt/stacks/geonetagent-collector/` rebuild; health OK | |
||||
| Guide | `docs/guide/device-model-catalog.md` | |
||||
|
||||
**Root cause stale massal:** banyak PC jarang lapor + bug owner (500) + GPO belum massal — bukan UI saja. |
||||
|
||||
@ -0,0 +1,64 @@
@@ -0,0 +1,64 @@
|
||||
# Device Model Catalog (SSOT) |
||||
|
||||
Manual mapping **Manufacturer + Model (WMI)** → **Series** (nama produk marketing). |
||||
|
||||
## Database |
||||
|
||||
| Tabel | Kunci unik | |
||||
|-------|------------| |
||||
| `device_model_catalog` | `manufacturer_normalized` + `model_normalized` | |
||||
|
||||
Migration: `infra/db/migrations/006_device_model_catalog.sql` |
||||
Pending series (nullable): `infra/db/migrations/007_device_model_catalog_pending_series.sql` |
||||
|
||||
Seed contoh (repo): `infra/db/seed/device_model_catalog.json` |
||||
|
||||
Normalisasi lookup: |
||||
- Manufacturer → uppercase, hapus suffix `Inc.`, `Ltd.`, dll. |
||||
- Model → uppercase, trim spasi |
||||
|
||||
Contoh: `LENOVO` + `82AU` → `Lenovo Legion 5-15IMH05` |
||||
|
||||
## API (Bearer token) |
||||
|
||||
| Method | Path | Fungsi | |
||||
|--------|------|--------| |
||||
| GET | `/api/v1/device-models` | Daftar mapping (`?q=` search, `?pending_only=true`) | |
||||
| GET | `/api/v1/device-models/lookup?manufacturer=&model=` | Resolve satu pasangan | |
||||
| POST | `/api/v1/device-models/sync-from-assets` | Auto-insert model dari tabel `assets` yang belum ada di catalog | |
||||
| POST | `/api/v1/device-models` | Tambah / upsert mapping | |
||||
| PATCH | `/api/v1/device-models/{id}` | Update series/notes | |
||||
| DELETE | `/api/v1/device-models/{id}` | Hapus mapping | |
||||
|
||||
POST body: |
||||
|
||||
```json |
||||
{ |
||||
"manufacturer": "LENOVO", |
||||
"model": "82AU", |
||||
"series": "Lenovo Legion 5-15IMH05", |
||||
"source": "manual", |
||||
"notes": "PSREF" |
||||
} |
||||
``` |
||||
|
||||
Asset API menambahkan field `series` (read-only, dari catalog) pada `GET /assets` dan `GET /assets/{id}`. |
||||
|
||||
## UI |
||||
|
||||
[https://collector.gisportal.id/ui/](https://collector.gisportal.id/ui/) |
||||
|
||||
1. Hubungkan dengan Bearer token |
||||
2. Klik **Model catalog** → **Sync dari assets** (isi otomatis manufacturer + model dari DB) |
||||
3. Isi kolom **Series** pada baris kuning (pending), atau gunakan form manual |
||||
4. Di detail asset tanpa series → **Tambah Series ke SSOT** |
||||
|
||||
## Deploy |
||||
|
||||
```bash |
||||
# Di PostgreSQL collector |
||||
psql -U geonet_collector -d geonetagent_dev -f infra/db/migrations/006_device_model_catalog.sql |
||||
psql -U geonet_collector -d geonetagent_dev -f infra/db/migrations/007_device_model_catalog_pending_series.sql |
||||
|
||||
# Rebuild + restart container geonet-collector di 10.100.1.24 |
||||
``` |
||||
@ -0,0 +1,31 @@
@@ -0,0 +1,31 @@
|
||||
-- Migration 006: SSOT lookup manufacturer + model → series (manual catalog) |
||||
-- Run on geonetagent_dev / geonetagent |
||||
|
||||
CREATE TABLE IF NOT EXISTS device_model_catalog ( |
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(), |
||||
manufacturer varchar(100) NOT NULL, |
||||
model varchar(100) NOT NULL, |
||||
manufacturer_normalized varchar(100) NOT NULL, |
||||
model_normalized varchar(100) NOT NULL, |
||||
series varchar(255) NOT NULL, |
||||
source varchar(50) NOT NULL DEFAULT 'manual', |
||||
notes text, |
||||
created_at timestamptz NOT NULL DEFAULT now(), |
||||
updated_at timestamptz NOT NULL DEFAULT now(), |
||||
CONSTRAINT uq_device_model_catalog_key UNIQUE (manufacturer_normalized, model_normalized) |
||||
); |
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_device_model_catalog_mfr_model |
||||
ON device_model_catalog (manufacturer_normalized, model_normalized); |
||||
|
||||
CREATE TRIGGER trg_device_model_catalog_updated_at |
||||
BEFORE UPDATE ON device_model_catalog |
||||
FOR EACH ROW EXECUTE PROCEDURE set_updated_at(); |
||||
|
||||
-- Contoh seed (opsional — hapus jika tidak perlu) |
||||
INSERT INTO device_model_catalog ( |
||||
manufacturer, model, manufacturer_normalized, model_normalized, series, source, notes |
||||
) VALUES ( |
||||
'LENOVO', '82AU', 'LENOVO', '82AU', |
||||
'Lenovo Legion 5-15IMH05', 'manual', 'PSREF / contoh SSOT' |
||||
) ON CONFLICT (manufacturer_normalized, model_normalized) DO NOTHING; |
||||
@ -0,0 +1,3 @@
@@ -0,0 +1,3 @@
|
||||
-- Migration 007: allow pending catalog rows (series belum diisi) |
||||
ALTER TABLE device_model_catalog |
||||
ALTER COLUMN series DROP NOT NULL; |
||||
@ -0,0 +1,9 @@
@@ -0,0 +1,9 @@
|
||||
[ |
||||
{ |
||||
"manufacturer": "LENOVO", |
||||
"model": "82AU", |
||||
"series": "Lenovo Legion 5-15IMH05", |
||||
"source": "manual", |
||||
"notes": "PSREF — contoh entri SSOT" |
||||
} |
||||
] |
||||
Loading…
Reference in new issue