Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Cherry-pick #21979 to 7.x: [Packetbeat] Create x-pack magefile #22545

Merged
merged 2 commits into from
Nov 11, 2020
Merged

Cherry-pick #21979 to 7.x: [Packetbeat] Create x-pack magefile #22545

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
32bc901
[Packetbeat] Create x-pack magefile (#21979)
Oct 27, 2020
77f84c7
Fix up changelog
Nov 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .ci/packaging.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ pipeline {
'x-pack/heartbeat',
// 'x-pack/journalbeat',
'x-pack/metricbeat',
// 'x-pack/packetbeat',
'x-pack/packetbeat',
'x-pack/winlogbeat'
)
}
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -588,6 +588,7 @@ same journal. {pull}18467[18467]

`host` metadata fields when processing network data from network tap or mirror
port. {pull}19209[19209]
- Change build process for x-pack distribution {pull}21979[21979]


*Functionbeat*
Expand Down
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ XPACK_SUFFIX=x-pack/
# PROJECTS_XPACK_PKG is a list of Beats that have independent packaging support
# in the x-pack directory (rather than having the OSS build produce both sets
# of artifacts). This will be removed once we complete the transition.
PROJECTS_XPACK_PKG=x-pack/auditbeat x-pack/dockerlogbeat x-pack/filebeat x-pack/heartbeat x-pack/metricbeat x-pack/winlogbeat
PROJECTS_XPACK_PKG=x-pack/auditbeat x-pack/dockerlogbeat x-pack/filebeat x-pack/heartbeat x-pack/metricbeat x-pack/winlogbeat x-pack/packetbeat
# PROJECTS_XPACK_MAGE is a list of Beats whose primary build logic is based in
# Mage. For compatibility with CI testing these projects support a subset of the
# makefile targets. After all Beats converge to primarily using Mage we can
Expand Down
323 changes: 6 additions & 317 deletions packetbeat/magefile.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,9 @@ package main

import (
"fmt"
"log"
"strings"
"time"

"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
"github.com/pkg/errors"

devtools "github.com/elastic/beats/v7/dev-tools/mage"
packetbeat "github.com/elastic/beats/v7/packetbeat/scripts/mage"
Expand All @@ -44,7 +40,7 @@ import (

func init() {
common.RegisterCheckDeps(Update)
unittest.RegisterPythonTestDeps(fieldsYML, Dashboards)
unittest.RegisterPythonTestDeps(packetbeat.FieldsYML, Dashboards)

devtools.BeatDescription = "Packetbeat analyzes network traffic and sends the data to Elasticsearch."
}
Expand All @@ -57,21 +53,7 @@ func Build() error {
// GolangCrossBuild build the Beat binary inside of the golang-builder.
// Do not use directly, use crossBuild instead.
func GolangCrossBuild() error {
if dep, found := crossBuildDeps[devtools.Platform.Name]; found {
mg.Deps(dep)
}

params := devtools.DefaultGolangCrossBuildArgs()
if flags, found := libpcapLDFLAGS[devtools.Platform.Name]; found {
params.Env = map[string]string{
"CGO_LDFLAGS": flags,
}
}
if flags, found := libpcapCFLAGS[devtools.Platform.Name]; found {
params.Env["CGO_CFLAGS"] = flags
}

return devtools.GolangCrossBuild(params)
return packetbeat.GolangCrossBuild()
}

// BuildGoDaemon builds the go-daemon binary (use crossBuildGoDaemon).
Expand Down Expand Up @@ -118,12 +100,12 @@ func Package() {
start := time.Now()
defer func() { fmt.Println("package ran for", time.Since(start)) }()

devtools.UseElasticBeatPackaging()
devtools.UseElasticBeatOSSPackaging()
devtools.PackageKibanaDashboardsFromBuildDir()
customizePackaging()
packetbeat.CustomizePackaging()

mg.Deps(Update)
mg.Deps(CrossBuild, CrossBuildXPack, CrossBuildGoDaemon)
mg.Deps(CrossBuild, CrossBuildGoDaemon)
mg.SerialDeps(devtools.Package, TestPackages)
}

Expand Down Expand Up @@ -151,27 +133,7 @@ func includeList() error {

// Fields generates fields.yml and fields.go files for the Beat.
func Fields() {
mg.Deps(libbeatAndPacketbeatCommonFieldsGo, protosFieldsGo)
mg.Deps(fieldsYML)
}

// libbeatAndPacketbeatCommonFieldsGo generates a fields.go containing both
// libbeat and packetbeat's common fields.
func libbeatAndPacketbeatCommonFieldsGo() error {
if err := devtools.GenerateFieldsYAML(); err != nil {
return err
}
return devtools.GenerateAllInOneFieldsGo()
}

// protosFieldsGo generates a fields.go for each protocol.
func protosFieldsGo() error {
return devtools.GenerateModuleFieldsGo("protos")
}

// fieldsYML generates the fields.yml file containing all fields.
func fieldsYML() error {
return devtools.GenerateFieldsYAML("protos")
packetbeat.Fields()
}

func fieldDocs() error {
Expand All @@ -182,276 +144,3 @@ func fieldDocs() error {
func Dashboards() error {
return devtools.KibanaDashboards("protos")
}

// -----------------------------------------------------------------------------
// Customizations specific to Packetbeat.
// - Config file contains an OS specific device name (affects darwin, windows).
// - Must compile libpcap or winpcap during cross-compilation.
// - On Linux libpcap is statically linked. Darwin and Windows are dynamic.

const (
libpcapURL = "https://s3.amazonaws.com/beats-files/deps/libpcap-1.8.1.tar.gz"
libpcapSHA256 = "673dbc69fdc3f5a86fb5759ab19899039a8e5e6c631749e48dcd9c6f0c83541e"
)

const (
linuxPcapLDFLAGS = "-L/libpcap/libpcap-1.8.1 -lpcap"
linuxPcapCFLAGS = "-I /libpcap/libpcap-1.8.1"
)

var libpcapLDFLAGS = map[string]string{
"linux/386": linuxPcapLDFLAGS,
"linux/amd64": linuxPcapLDFLAGS,
"linux/arm64": linuxPcapLDFLAGS,
"linux/armv5": linuxPcapLDFLAGS,
"linux/armv6": linuxPcapLDFLAGS,
"linux/armv7": linuxPcapLDFLAGS,
"linux/mips": linuxPcapLDFLAGS,
"linux/mipsle": linuxPcapLDFLAGS,
"linux/mips64": linuxPcapLDFLAGS,
"linux/mips64le": linuxPcapLDFLAGS,
"linux/ppc64le": linuxPcapLDFLAGS,
"linux/s390x": linuxPcapLDFLAGS,
"darwin/amd64": "-lpcap",
"windows/amd64": "-L /libpcap/win/WpdPack/Lib/x64 -lwpcap",
"windows/386": "-L /libpcap/win/WpdPack/Lib -lwpcap",
}

var libpcapCFLAGS = map[string]string{
"linux/386": linuxPcapCFLAGS,
"linux/amd64": linuxPcapCFLAGS,
"linux/arm64": linuxPcapCFLAGS,
"linux/armv5": linuxPcapCFLAGS,
"linux/armv6": linuxPcapCFLAGS,
"linux/armv7": linuxPcapCFLAGS,
"linux/mips": linuxPcapCFLAGS,
"linux/mipsle": linuxPcapCFLAGS,
"linux/mips64": linuxPcapCFLAGS,
"linux/mips64le": linuxPcapCFLAGS,
"linux/ppc64le": linuxPcapCFLAGS,
"linux/s390x": linuxPcapCFLAGS,
"windows/amd64": "-I /libpcap/win/WpdPack/Include",
"windows/386": "-I /libpcap/win/WpdPack/Include",
}

var crossBuildDeps = map[string]func() error{
"linux/386": buildLibpcapLinux386,
"linux/amd64": buildLibpcapLinuxAMD64,
"linux/arm64": buildLibpcapLinuxARM64,
"linux/armv5": buildLibpcapLinuxARMv5,
"linux/armv6": buildLibpcapLinuxARMv6,
"linux/armv7": buildLibpcapLinuxARMv7,
"linux/mips": buildLibpcapLinuxMIPS,
"linux/mipsle": buildLibpcapLinuxMIPSLE,
"linux/mips64": buildLibpcapLinuxMIPS64,
"linux/mips64le": buildLibpcapLinuxMIPS64LE,
"linux/ppc64le": buildLibpcapLinuxPPC64LE,
"linux/s390x": buildLibpcapLinuxS390x,
"windows/amd64": installLibpcapWindowsAMD64,
"windows/386": installLibpcapWindows386,
}

// buildLibpcapFromSource builds libpcap from source because the library needs
// to be compiled with -fPIC.
// See https://github.com/elastic/beats/v7/pull/4217.
func buildLibpcapFromSource(params map[string]string) error {
tarFile, err := devtools.DownloadFile(libpcapURL, "/libpcap")
if err != nil {
return errors.Wrap(err, "failed to download libpcap source")
}

if err = devtools.VerifySHA256(tarFile, libpcapSHA256); err != nil {
return err
}

if err = devtools.Extract(tarFile, "/libpcap"); err != nil {
return errors.Wrap(err, "failed to extract libpcap")
}

var configureArgs []string
for k, v := range params {
if strings.HasPrefix(k, "-") {
delete(params, k)
configureArgs = append(configureArgs, k+"="+v)
}
}

// Use sh -c here because sh.Run does not expose a way to change the CWD.
// This command only runs in Linux so this is fine.
return sh.RunWith(params, "sh", "-c",
"cd /libpcap/libpcap-1.8.1 && "+
"./configure --enable-usb=no --enable-bluetooth=no --enable-dbus=no "+strings.Join(configureArgs, " ")+"&& "+
"make")
}

func buildLibpcapLinux386() error {
return buildLibpcapFromSource(map[string]string{
"CFLAGS": "-m32",
"LDFLAGS": "-m32",
})
}

func buildLibpcapLinuxAMD64() error {
return buildLibpcapFromSource(map[string]string{})
}

func buildLibpcapLinuxARM64() error {
return buildLibpcapFromSource(map[string]string{
"--host": "aarch64-unknown-linux-gnu",
"--with-pcap": "linux",
})
}

func buildLibpcapLinuxARMv5() error {
return buildLibpcapFromSource(map[string]string{
"--host": "arm-linux-gnueabi",
"--with-pcap": "linux",
})
}

func buildLibpcapLinuxARMv6() error {
return buildLibpcapFromSource(map[string]string{
"--host": "arm-linux-gnueabi",
"--with-pcap": "linux",
})
}

func buildLibpcapLinuxARMv7() error {
return buildLibpcapFromSource(map[string]string{
"--host": "arm-linux-gnueabihf",
"--with-pcap": "linux",
})
}

func buildLibpcapLinuxMIPS() error {
return buildLibpcapFromSource(map[string]string{
"--host": "mips-unknown-linux-gnu",
"--with-pcap": "linux",
})
}

func buildLibpcapLinuxMIPSLE() error {
return buildLibpcapFromSource(map[string]string{
"--host": "mipsle-unknown-linux-gnu",
"--with-pcap": "linux",
})
}

func buildLibpcapLinuxMIPS64() error {
return buildLibpcapFromSource(map[string]string{
"--host": "mips64-unknown-linux-gnu",
"--with-pcap": "linux",
})
}

func buildLibpcapLinuxMIPS64LE() error {
return buildLibpcapFromSource(map[string]string{
"--host": "mips64le-unknown-linux-gnu",
"--with-pcap": "linux",
})
}

func buildLibpcapLinuxPPC64LE() error {
return buildLibpcapFromSource(map[string]string{
"--host": "powerpc64le-linux-gnu",
"--with-pcap": "linux",
})
}

func buildLibpcapLinuxS390x() error {
return buildLibpcapFromSource(map[string]string{
"--host": "s390x-ibm-linux-gnu",
"--with-pcap": "linux",
})
}

func installLibpcapWindowsAMD64() error {
mg.SerialDeps(installWinpcap, generateWin64StaticWinpcap)
return nil
}

func installLibpcapWindows386() error {
return installWinpcap()
}

func installWinpcap() error {
log.Println("Install Winpcap")
const wpdpackURL = "https://www.winpcap.org/install/bin/WpdPack_4_1_2.zip"

winpcapZip, err := devtools.DownloadFile(wpdpackURL, "/")
if err != nil {
return err
}

if err = devtools.Extract(winpcapZip, "/libpcap/win"); err != nil {
return err
}

return nil
}

func generateWin64StaticWinpcap() error {
log.Println(">> Generating 64-bit winpcap static lib")

// Notes: We are using absolute path to make sure the files
// are available for x-pack build.
// Ref: https://github.com/elastic/beats/v7/issues/1259
defer devtools.DockerChown(devtools.MustExpand("{{elastic_beats_dir}}/{{.BeatName}}/lib"))
return devtools.RunCmds(
// Requires mingw-w64-tools.
[]string{"gendef", devtools.MustExpand("{{elastic_beats_dir}}/{{.BeatName}}/lib/windows-64/wpcap.dll")},
[]string{"mv", "wpcap.def", devtools.MustExpand("{{ elastic_beats_dir}}/{{.BeatName}}/lib/windows-64/wpcap.def")},
[]string{"x86_64-w64-mingw32-dlltool", "--as-flags=--64",
"-m", "i386:x86-64", "-k",
"--output-lib", "/libpcap/win/WpdPack/Lib/x64/libwpcap.a",
"--input-def", devtools.MustExpand("{{elastic_beats_dir}}/{{.BeatName}}/lib/windows-64/wpcap.def")},
)
}

// customizePackaging modifies the device in the configuration files based on
// the target OS.
func customizePackaging() {
var (
configYml = devtools.PackageFile{
Mode: 0600,
Source: "{{.PackageDir}}/{{.BeatName}}.yml",
Config: true,
Dep: func(spec devtools.PackageSpec) error {
c := packetbeat.ConfigFileParams()
c.ExtraVars["GOOS"] = spec.OS
c.ExtraVars["GOARCH"] = spec.MustExpand("{{.GOARCH}}")
return devtools.Config(devtools.ShortConfigType, c, spec.MustExpand("{{.PackageDir}}"))
},
}
referenceConfigYml = devtools.PackageFile{
Mode: 0644,
Source: "{{.PackageDir}}/{{.BeatName}}.reference.yml",
Dep: func(spec devtools.PackageSpec) error {
c := packetbeat.ConfigFileParams()
c.ExtraVars["GOOS"] = spec.OS
c.ExtraVars["GOARCH"] = spec.MustExpand("{{.GOARCH}}")
return devtools.Config(devtools.ReferenceConfigType, c, spec.MustExpand("{{.PackageDir}}"))
},
}
)

for _, args := range devtools.Packages {
for _, pkgType := range args.Types {
switch pkgType {
case devtools.TarGz, devtools.Zip:
args.Spec.ReplaceFile("{{.BeatName}}.yml", configYml)
args.Spec.ReplaceFile("{{.BeatName}}.reference.yml", referenceConfigYml)
case devtools.Deb, devtools.RPM, devtools.DMG:
args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.yml", configYml)
args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.reference.yml", referenceConfigYml)
case devtools.Docker:
args.Spec.ExtraVar("linux_capabilities", "cap_net_raw,cap_net_admin=eip")
default:
panic(errors.Errorf("unhandled package type: %v", pkgType))
}

// Match the first package type then continue.
break
}
}
}
Loading