Skip to content

feature: Support for auto parsing http gzip response content #260 #301

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: feature/1.6.0
Choose a base branch
from
Open

feature: Support for auto parsing http gzip response content #260 #301

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6c59356
feature: Support for auto parsing http gzip response content #260
haolipeng Mar 9, 2025
059fb5a
chore: update github workflow actions version
hengyoush Mar 11, 2025
e3cdab5
fix: #300 (#302)
hengyoush Mar 11, 2025
760a7eb
Merge branch 'hengyoush:main' into main
haolipeng Mar 13, 2025
2bd85ff
add test_http_gzip.sh to github workflows
haolipeng Mar 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 40 additions & 29 deletions .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@ jobs:
build:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
- uses: actions/checkout@v4.2.2
with:
submodules: recursive

- name: Set up Go
uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5
uses: actions/setup-go@v5.3.0
with:
go-version: '1.22.4'

Expand All @@ -48,7 +48,7 @@ jobs:
# run: make test

- name: Store executable
uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4
uses: actions/upload-artifact@v4.6.1
with:
name: kyanos
path: kyanos
Expand Down Expand Up @@ -80,18 +80,18 @@ jobs:
timeout-minutes: 30
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
- uses: actions/checkout@v4.2.2
with:
submodules: recursive

- name: Retrieve stored kyanos executable
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
uses: actions/download-artifact@v4.1.9
with:
name: kyanos
path: kyanos

- name: Provision LVH VMs
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
test-name: kyanos-test
image-version: ${{ matrix.kernel }}
Expand All @@ -116,7 +116,7 @@ jobs:

- name: copy btf file
if: ${{ startsWith(matrix.kernel, '4.') }}
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
Expand All @@ -131,7 +131,7 @@ jobs:
# sudo cp $btf_file /var/lib/ptcpdump/btf/vmlinux

- name: Install dependencies
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
Expand Down Expand Up @@ -163,7 +163,7 @@ jobs:
docker-compose --version

- name: Test Truncated Data parsing
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
if: ${{ !contains(fromJSON('["4.19-20240912.022020", "5.4-20240912.022020"]'), matrix.kernel) }}
with:
provision: 'false'
Expand All @@ -180,7 +180,7 @@ jobs:
popd

- name: Test Kafka
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
Expand All @@ -196,7 +196,7 @@ jobs:
popd

- name: Test CAP_BPF privilege check
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
if: ${{ !contains(fromJSON('["4.19-20240912.022020", "5.4-20240912.022020"]'), matrix.kernel) }}
with:
provision: 'false'
Expand All @@ -209,7 +209,7 @@ jobs:
popd

- name: Test CAP_SYS_ADMIN privilege check
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
if: contains(fromJSON('["4.19-20240912.022020", "5.4-20240912.022020"]'), matrix.kernel)
with:
provision: 'false'
Expand All @@ -222,7 +222,7 @@ jobs:
popd

- name: Test RocketMQ
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
Expand All @@ -238,7 +238,7 @@ jobs:
popd

- name: Test filter by comm
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
Expand All @@ -252,9 +252,21 @@ jobs:
bash /host/testdata/test_filter_by_comm.sh 'sudo /host/kyanos/kyanos $kyanos_log_option'
fi
popd

- name: Test http with gzip
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
set -ex
uname -a
cat /etc/issue
pushd /host
bash /host/testdata/test_http_gzip.sh 'sudo /host/kyanos/kyanos $kyanos_log_option'
popd

- name: Test gotls
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
Expand All @@ -270,7 +282,7 @@ jobs:
popd

- name: Test https
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
Expand All @@ -286,7 +298,7 @@ jobs:
popd

- name: Test side
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
Expand All @@ -302,7 +314,7 @@ jobs:
popd

- name: Test mysql
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
Expand All @@ -318,7 +330,7 @@ jobs:
popd

- name: Test base
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
Expand All @@ -331,9 +343,8 @@ jobs:
bash /host/testdata/test_base.sh 'sudo /host/kyanos/kyanos $kyanos_log_option'
fi


- name: Test filter by l3/l4 info
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
Expand All @@ -347,7 +358,7 @@ jobs:
fi

- name: Test kern evt
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
Expand All @@ -361,7 +372,7 @@ jobs:
fi

- name: Test test docker filter by container id
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
Expand All @@ -375,7 +386,7 @@ jobs:
fi

- name: Test test docker filter by container name
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
Expand All @@ -389,7 +400,7 @@ jobs:
fi

- name: Test filter by pid
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
Expand All @@ -403,7 +414,7 @@ jobs:
fi

- name: Test test containerd filter by container name
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
Expand All @@ -417,7 +428,7 @@ jobs:
fi

- name: Test test containerd filter by container id
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
Expand All @@ -431,7 +442,7 @@ jobs:
fi

- name: Test redis
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
Expand All @@ -446,7 +457,7 @@ jobs:

- name: Test k8s
if: ${{ startsWith(matrix.kernel, '6.') }}
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
uses: cilium/little-vm-helper@c44c1221b104ee02ec0235211f7ace3c88eb11a2 # v0.0.19
with:
provision: 'false'
cmd: |
Expand Down
10 changes: 5 additions & 5 deletions agent/protocol/fakedata.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ func MakeNewFakeData(size uint32) ([]byte, bool) {
copy(buf, fakeDataMarkPrefix)
size -= uint32(fakeDataMarkLen)
lenByte := []byte{byte(size >> 24), byte(size >> 16), byte(size >> 8), byte(size)}
buf[fakeDataMarkLen] = lenByte[0]
buf[fakeDataMarkLen+1] = lenByte[1]
buf[fakeDataMarkLen+2] = lenByte[2]
buf[fakeDataMarkLen+3] = lenByte[3]
buf[len(fakeDataMarkPrefix)] = lenByte[0]
buf[len(fakeDataMarkPrefix)+1] = lenByte[1]
buf[len(fakeDataMarkPrefix)+2] = lenByte[2]
buf[len(fakeDataMarkPrefix)+3] = lenByte[3]
return buf, true
}
}
Expand All @@ -37,6 +37,6 @@ func getFakeDataSize(buf []byte, pos int) uint32 {
if len(buf) < fakeDataMarkLen {
return 0
} else {
return uint32(buf[fakeDataMarkLen])<<24 | uint32(buf[fakeDataMarkLen+1])<<16 | uint32(buf[fakeDataMarkLen+2])<<8 | uint32(buf[fakeDataMarkLen+3])
return uint32(buf[len(fakeDataMarkPrefix)])<<24 | uint32(buf[len(fakeDataMarkPrefix)+1])<<16 | uint32(buf[len(fakeDataMarkPrefix)+2])<<8 | uint32(buf[len(fakeDataMarkPrefix)+3])
}
}
37 changes: 36 additions & 1 deletion agent/protocol/http.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package protocol

import (
"bufio"
"bytes"
"compress/gzip"
"fmt"
"io"
"kyanos/agent/buffer"
Expand Down Expand Up @@ -136,6 +138,17 @@ func (h *HTTPStreamParser) ParseResponse(buf string, messageType MessageType, ti
return h.handleReadResponseError(err, buf, streamBuffer, messageType, timestamp, seq)
}

gzipFlag := false
contentEncodingList := resp.Header["Content-Encoding"]
if len(contentEncodingList) > 0 {
for _, v := range contentEncodingList {
if v == "gzip" {
gzipFlag = true
break
}
}
}

respBody, err := io.ReadAll(resp.Body)
if err != nil {
return h.handleReadBodyError(err, buf, streamBuffer, messageType, timestamp, seq)
Expand All @@ -154,6 +167,7 @@ func (h *HTTPStreamParser) ParseResponse(buf string, messageType MessageType, ti
&ParsedHttpResponse{
FrameBase: NewFrameBase(timestamp, readIndex, seq),
buf: []byte(buf[:readIndex]),
isGzip: gzipFlag,
},
}
parseResult.ParseState = Success
Expand Down Expand Up @@ -260,7 +274,8 @@ func (req *ParsedHttpRequest) StreamId() StreamId {

type ParsedHttpResponse struct {
FrameBase
buf []byte
isGzip bool
buf []byte
}

func (resp *ParsedHttpResponse) FormatToSummaryString() string {
Expand All @@ -271,6 +286,26 @@ func (resp *ParsedHttpResponse) Status() ResponseStatus {
}

func (resp *ParsedHttpResponse) FormatToString() string {
if resp.isGzip {
// 创建一个bytes.Reader
bytesReader := bytes.NewReader(resp.buf)

// 创建gzip.Reader
gzipReader, err := gzip.NewReader(bytesReader)
if err != nil {
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

resp.buf contains the response headers, so you CAN NOT gzip decompress the entire resp.buf as this will fail. Hint: If you debug to this point, you'll find that err is not nil.

return string(resp.buf)
}
defer gzipReader.Close()

// 读取解压后的内容
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pls use English

decompressed, err := io.ReadAll(gzipReader)
if err != nil {
return string(resp.buf)
}

// 转换为string并返回
return string(decompressed)
}
return string(resp.buf)
}
func (resp *ParsedHttpResponse) IsReq() bool {
Expand Down
Loading
Loading