Merge pull request #2533 from akto-api-security/feat/ui-schema-conformance

gauravakto · web-flow · commit f5f61e1edc56 · 2025-05-27T10:46:30.000+05:30
Feat/UI schema conformance
diff --git a/apps/dashboard/src/main/java/com/akto/action/threat_detection/MaliciousPayloadsResponse.java b/apps/dashboard/src/main/java/com/akto/action/threat_detection/MaliciousPayloadsResponse.java
@@ -3,12 +3,28 @@
 public class MaliciousPayloadsResponse {
     
     private String orig;
+    private String metadata;
     private long ts;
+
+    public MaliciousPayloadsResponse(String orig, String metadata, long ts) {
+        this.orig = orig;
+        this.metadata = metadata;
+        this.ts = ts;
+    }
+
     public MaliciousPayloadsResponse(String orig, long ts) {
         this.orig = orig;
         this.ts = ts;
     }
-    
+
+    public String getMetadata() {
+        return metadata;
+    }
+
+    public void setMetadata(String metadata) {
+        this.metadata = metadata;
+    }
+
     public String getOrig() {
         return orig;
     }
diff --git a/apps/dashboard/src/main/java/com/akto/action/threat_detection/ThreatActorAction.java b/apps/dashboard/src/main/java/com/akto/action/threat_detection/ThreatActorAction.java
@@ -225,6 +225,7 @@ public String fetchAggregateMaliciousRequests() {
                             smr ->
                                 new MaliciousPayloadsResponse(
                                     smr.getOrig(),
+                                    smr.getMetadata(),
                                     smr.getTs()))
                         .collect(Collectors.toList());
               });
diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/SampleDataList.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/SampleDataList.js
@@ -1,5 +1,7 @@
 import React, { useState, useEffect } from 'react'
 import {
+  Banner,
+  List,
   Text,
   VerticalStack,
   HorizontalStack, Box, LegacyCard, HorizontalGrid,
@@ -8,6 +10,33 @@ import SampleDataComponent from './SampleDataComponent';
 import SampleData from './SampleData';
 import func from '../../../../util/func';
 
+function SchemaValidationError({ sampleData}) {
+    if (!sampleData || !sampleData?.metadata) {
+        return null;
+    }
+    const schemaErrors = JSON.parse(sampleData?.metadata)?.schemaErrors || [];
+    if (schemaErrors.length === 0) {
+        return null;
+    }
+
+
+    return (
+        <VerticalStack gap={"4"}>
+            <Banner
+                title="Schema Validation Errors"
+                status="critical"
+            >
+                <List type="bullet">
+                    {schemaErrors?.map((error, index) => {
+                        return <List.Item key={index}>{error?.message}</List.Item>
+                    })}
+                </List>
+            </Banner>
+
+        </VerticalStack>
+    )
+}
+
 function SampleDataList(props) {
 
     const {showDiff, sampleData, heading, minHeight, vertical, isVulnerable, isNewDiff, metadata} = props;
@@ -20,6 +49,7 @@ function SampleDataList(props) {
   
     return (
       <VerticalStack gap="3">
+        <SchemaValidationError sampleData={sampleData[Math.min(page, sampleData.length - 1)]} />
         <HorizontalStack align='space-between'>
           <HorizontalStack gap="2">
             <Text variant='headingMd'>
diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiDetails.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiDetails.jsx
@@ -55,7 +55,6 @@ function ApiDetails(props) {
     const [prompts, setPrompts] = useState([])
     const [isGptScreenActive, setIsGptScreenActive] = useState(false)
     const [loading, setLoading] = useState(false)
-    const [badgeActive, setBadgeActive] = useState(false)
     const [showMoreActions, setShowMoreActions] = useState(false)
     const setSelectedSampleApi = PersistStore(state => state.setSelectedSampleApi)
     const [disabledTabs, setDisabledTabs] = useState([])
@@ -257,36 +256,6 @@ function ApiDetails(props) {
             setTimeout(() => {
                 setLoading(false)
             }, 100)
-            await api.loadParamsOfEndpoint(apiCollectionId, endpoint, method).then(resp => {
-                api.loadSensitiveParameters(apiCollectionId, endpoint, method).then(allSensitiveFields => {
-                    allSensitiveFields.data.endpoints.filter(x => x.sensitive).forEach(sensitive => {
-                        let index = resp.data.params.findIndex(x =>
-                            x.param === sensitive.param &&
-                            x.isHeader === sensitive.isHeader &&
-                            x.responseCode === sensitive.responseCode
-                        )
-
-                        if (index > -1 && !sensitive.subType) {
-                            resp.data.params[index].savedAsSensitive = true
-                            if (!resp.data.params[index].subType) {
-                                resp.data.params[index].subType = { "name": "CUSTOM" }
-                            } else {
-                                resp.data.params[index].subType = JSON.parse(JSON.stringify(resp.data.params[index].subType))
-                            }
-                        }
-                    })
-
-                    try {
-                        resp.data.params?.forEach(x => {
-                            if (!values?.skipList.includes(x.subTypeString) && !x?.savedAsSensitive && !x?.sensitive) {
-                                x.nonSensitiveDataType = true
-                            }
-                        })
-                    } catch (e){
-                    }
-                    setParamList(resp.data.params)
-                })
-            })
             fetchStats(apiCollectionId, endpoint, method)
             fetchDistributionData(); // Fetch distribution data
         }
@@ -322,10 +291,6 @@ function ApiDetails(props) {
         func.setToast(true, false, "Triggered tests successfully!")
     }
 
-    const badgeClicked = () => {
-        setBadgeActive(true)
-    }
-
     useEffect(() => {
         if (
             (localCategoryMap && Object.keys(localCategoryMap).length > 0) &&
@@ -456,11 +421,8 @@ function ApiDetails(props) {
     const SchemaTab = {
         id: 'schema',
         content: "Schema",
-        component: paramList.length > 0 && <Box paddingBlockStart={"4"}> 
+        component:  <Box paddingBlockStart={"4"}> 
             <ApiSchema
-                data={paramList} 
-                badgeActive={badgeActive}
-                setBadgeActive={setBadgeActive}
                 apiInfo={{
                     apiCollectionId: apiDetail.apiCollectionId,
                     url: apiDetail.endpoint,
@@ -611,8 +573,6 @@ function ApiDetails(props) {
                         data={newData}
                         headers={headers}
                         getStatus={statusFunc}
-                        isBadgeClickable={true}
-                        badgeClicked={badgeClicked}
                     />
                 </HorizontalStack>
             </VerticalStack>
diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiSchema.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiSchema.jsx
@@ -4,6 +4,7 @@ import { ChevronDownMinor, ChevronUpMinor } from "@shopify/polaris-icons"
 import func from "@/util/func"
 import transform from "../transform";
 import { useNavigate } from "react-router-dom";
+import api from "../api";
 
 function prepareTableData (data, handleBadgeClick) {
     let sensitivePayload = []
@@ -70,11 +71,12 @@ function prepareTableData (data, handleBadgeClick) {
 }
 
 function ApiSingleSchema(props) {
-    const { data, title, badgeActive, setBadgeActive } = props;
+    const { data, title } = props;
 
     const [open, setOpen] = useState(true);
     const handleToggle = useCallback(() => setOpen((open) => !open), []);
     const [isHeader, setIsHeader] = useState(true)
+    const [badgeActive, setBadgeActive] = useState(true)
 
     const [dataObj,setDataObj] = useState({
         headerData: [],
@@ -83,7 +85,7 @@ function ApiSingleSchema(props) {
     });
     useEffect(()=>{
         setDataObj(prepareTableData(data, props.handleBadgeClick));
-    },[])
+    },[data])
     const headerCount = dataObj?.headerData?.length
     const payloadCount = dataObj?.payloadData?.length
 
@@ -149,14 +151,61 @@ function ApiSingleSchema(props) {
     )
 }
 
+
 function ApiSchema(props) {
 
-    const { data, badgeActive, setBadgeActive, apiInfo } = props
+    const {badgeActive, setBadgeActive, apiInfo } = props
     const navigate = useNavigate()
+    const [payloadData, setPayloadData] = useState({
+        reqData: [],
+        resData: []
+    })
 
-    let reqData = data.filter((item) => item.responseCode === -1)
-    let resData = data.filter((item) => item.responseCode !== -1)
+    async function fetchData() {
+        const { apiCollectionId, url, method } = apiInfo;
+        await api.loadParamsOfEndpoint(apiCollectionId, url, method).then(resp => {
+            api.loadSensitiveParameters(apiCollectionId, url, method).then(allSensitiveFields => {
+                allSensitiveFields.data.endpoints.filter(x => x.sensitive).forEach(sensitive => {
+                    let index = resp.data.params.findIndex(x =>
+                        x.param === sensitive.param &&
+                        x.isHeader === sensitive.isHeader &&
+                        x.responseCode === sensitive.responseCode
+                    )
+
+                    if (index > -1 && !sensitive.subType) {
+                        resp.data.params[index].savedAsSensitive = true
+                        if (!resp.data.params[index].subType) {
+                            resp.data.params[index].subType = { "name": "CUSTOM" }
+                        } else {
+                            resp.data.params[index].subType = JSON.parse(JSON.stringify(resp.data.params[index].subType))
+                        }
+                    }
+                })
 
+                try {
+                    resp.data.params?.forEach(x => {
+                        if (!values?.skipList.includes(x.subTypeString) && !x?.savedAsSensitive && !x?.sensitive) {
+                            x.nonSensitiveDataType = true
+                        }
+                    })
+                } catch (e) {
+                }
+                let reqData = resp.data.params.filter((item) => item.responseCode === -1)
+                let resData = resp.data.params.filter((item) => item.responseCode !== -1)
+
+                setPayloadData({
+                    reqData: reqData,
+                    resData: resData
+                })
+            })
+        })
+    }
+
+    useEffect(() => {
+        if (apiInfo) {
+            fetchData()
+        }
+    }, [apiInfo])
     const handleBadgeClick = (datatype, position) => {
         const navUrl = "/dashboard/observe/sensitive/" + datatype.toUpperCase() + "/" + apiInfo.apiCollectionId + "/" + btoa(apiInfo.url + " " + apiInfo.method)
         navigate(navUrl)
@@ -166,7 +215,7 @@ function ApiSchema(props) {
         <VerticalStack gap="2">
             {
                 ['Request', 'Response'].map((type, index) => {
-                    return <ApiSingleSchema handleBadgeClick={handleBadgeClick} title={type} key={type} data={index == 0 ? reqData : resData} badgeActive={badgeActive} setBadgeActive={setBadgeActive}/>
+                    return <ApiSingleSchema handleBadgeClick={handleBadgeClick} title={type} key={type} data={index == 0 ? payloadData.reqData : payloadData.resData} badgeActive={badgeActive} setBadgeActive={setBadgeActive}/>
                 })
             }
 
diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/ThreatDetectionPage.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/ThreatDetectionPage.jsx
@@ -108,6 +108,8 @@ function ThreatDetectionPage() {
                 setShowDetails(true)
                 setMoreInfoData({
                     url: data.url,
+                    method: data.method,
+                    apiCollectionId: data.apiCollectionId,
                     templateId: data.filterId,
                 })
             } else {
diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/SampleDetails.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/SampleDetails.jsx
@@ -8,6 +8,7 @@ import testingApi from "../../testing/api"
 import MarkdownViewer from "../../../components/shared/MarkdownViewer";
 import TooltipText from "../../../components/shared/TooltipText";
 import ActivityTracker from "../../dashboard/components/ActivityTracker";
+import ApiSchema from "../../observe/api_collections/ApiSchema";
 
 function SampleDetails(props) {
     const { showDetails, setShowDetails, data, title, moreInfoData, threatFiltersMap } = props
@@ -80,21 +81,36 @@ function SampleDetails(props) {
         component: <ActivityTracker latestActivity={latestActivity} />
     }
 
+
+    const SchemaTab = {
+        id: 'schema',
+        content: "Schema",
+        component:  <Box paddingBlockStart={"4"}> 
+            <ApiSchema
+                apiInfo={{
+                    apiCollectionId: moreInfoData?.apiCollectionId,
+                    url: moreInfoData?.url,
+                    method: moreInfoData?.method
+                }}
+            />
+        </Box>
+    }
+
     const ValuesTab = {
         id: 'values',
         content: "Values",
         component: (
-        <Box paddingBlockStart={3} paddingInlineEnd={4} paddingInlineStart={4}>
-            <SampleDataList
-                key="Sample values"
-                heading={"Attempt"}
-                minHeight={"30vh"}
-                vertical={true}
-                sampleData={data?.map((result) => {
-                    return {message:result.orig, highlightPaths:[]}
-                })}
-            />
-        </Box>)
+            <Box paddingBlockStart={3} paddingInlineEnd={4} paddingInlineStart={4}>
+                <SampleDataList
+                    key="Sample values"
+                    heading={"Attempt"}
+                    minHeight={"30vh"}
+                    vertical={true}
+                    sampleData={data?.map((result) => {
+                        return { message: result.orig, highlightPaths: [], metadata: result.metadata }
+                    })}
+                />
+            </Box>)
     }
 
     const remediationTab = remediationText.length > 0 && {
diff --git a/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/ThreatActorService.java b/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/ThreatActorService.java
@@ -1,6 +1,7 @@
 package com.akto.threat.backend.service;
 
 import com.akto.dto.HttpResponseParams;
+import com.akto.proto.generated.threat_detection.message.sample_request.v1.Metadata;
 import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.DailyActorsCountResponse;
 import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.FetchMaliciousEventsRequest;
 import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.FetchMaliciousEventsResponse;
@@ -17,9 +18,11 @@
 import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.ThreatActorByCountryRequest;
 import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.ThreatActorByCountryResponse;
 import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.ListThreatActorResponse.ActivityData;
+import com.akto.ProtoMessageUtils;
 import com.akto.threat.backend.constants.MongoDBCollection;
 import com.akto.threat.backend.db.ActorInfoModel;
 import com.akto.threat.backend.db.SplunkIntegrationModel;
+import com.google.protobuf.TextFormat;
 import com.mongodb.client.FindIterable;
 import com.mongodb.client.MongoClient;
 import com.mongodb.client.MongoCollection;
@@ -378,6 +381,19 @@ public ThreatActivityTimelineResponse getThreatActivityTimeline(String accountId
         return ThreatActivityTimelineResponse.newBuilder().addAllThreatActivityTimeline(timeline).build();
   }
 
+  private String fetchMetadataString(Document doc){
+    String metadataStr = doc.getString("metadata");
+    Metadata.Builder metadataBuilder = Metadata.newBuilder();
+    try {
+      TextFormat.getParser().merge(metadataStr, metadataBuilder);
+    } catch (Exception e) {
+      return metadataStr;
+    }
+    Metadata metadataProto = metadataBuilder.build();
+    metadataStr = ProtoMessageUtils.toString(metadataProto).orElse("");
+    return metadataStr;
+  }
+
   public FetchMaliciousEventsResponse fetchAggregateMaliciousRequests(
       String accountId, FetchMaliciousEventsRequest request) {
 
@@ -392,6 +408,7 @@ public FetchMaliciousEventsResponse fetchAggregateMaliciousRequests(
             maliciousPayloadsResponse.add(
                 FetchMaliciousEventsResponse.MaliciousPayloadsResponse.newBuilder().
                 setOrig(HttpResponseParams.getSampleStringFromProtoString(doc.getString("latestApiOrig"))).
+                setMetadata(fetchMetadataString(doc)).
                 setTs(doc.getLong("detectedAt")).build());
         }
     } else {
@@ -401,6 +418,7 @@ public FetchMaliciousEventsResponse fetchAggregateMaliciousRequests(
             maliciousPayloadsResponse.add(
                 FetchMaliciousEventsResponse.MaliciousPayloadsResponse.newBuilder().
                 setOrig(HttpResponseParams.getSampleStringFromProtoString(doc.getString("orig"))).
+                setMetadata(fetchMetadataString(doc)).
                 setTs(doc.getLong("requestTime")).build());
         }
     }
diff --git a/libs/utils/src/main/java/com/akto/log/LoggerMaker.java b/libs/utils/src/main/java/com/akto/log/LoggerMaker.java
@@ -37,6 +37,7 @@ public class LoggerMaker  {
         System.setProperty(SimpleLogger.DEFAULT_LOG_LEVEL_KEY, System.getenv().getOrDefault("AKTO_LOG_LEVEL", "WARN"));
         System.setProperty("org.slf4j.simpleLogger.log.org.apache.kafka", "ERROR");
         System.setProperty("org.slf4j.simpleLogger.log.io.lettuce", "ERROR");
+        System.setProperty("org.slf4j.simpleLogger.log.org.mongodb", "ERROR");
         System.setProperty("org.slf4j.simpleLogger.log.io.netty", "ERROR");
         System.setProperty("org.slf4j.simpleLogger.log.org.flywaydb", "ERROR");
         System.out.printf("AKTO_LOG_LEVEL is set to: %s \n", System.getProperty(SimpleLogger.DEFAULT_LOG_LEVEL_KEY));
diff --git a/protobuf/threat_detection/service/dashboard_service/v1/service.proto b/protobuf/threat_detection/service/dashboard_service/v1/service.proto
