From d3f00c9011f025168d43cf6b597ff867632fdbc9 Mon Sep 17 00:00:00 2001 From: davidbabono Date: Sat, 20 Jan 2024 19:33:51 -0500 Subject: [PATCH] minlib tech with optimized rebuild --- cos | 16 +++-- src/Makefile | 7 ++ src/components/Makefile | 8 +++ src/components/implementation/Makefile | 5 ++ src/components/implementation/Makefile.subdir | 5 ++ .../implementation/Makefile.subsubdir | 17 +++-- src/components/implementation/comp_x86_64.ld | 2 +- .../implementation/pong/pingpong/Makefile | 2 + src/components/interface/Makefile | 13 ++++ src/components/interface/Makefile.subsubdir | 3 +- src/components/lib/Makefile | 16 +++++ src/components/lib/Makefile.lib | 2 +- src/composer/src/build.rs | 70 +++++++++++++++++-- src/composer/src/main.rs | 7 +- src/composer/src/passes.rs | 2 +- src/composer/src/resources.rs | 2 +- src/composer/src/syshelpers.rs | 4 +- 17 files changed, 158 insertions(+), 23 deletions(-) diff --git a/cos b/cos index b9e5037af2..45583ac173 100755 --- a/cos +++ b/cos @@ -6,7 +6,8 @@ script= name= binary= arch= - +minlib= + initialize() { case ${arch} in @@ -62,10 +63,16 @@ compose() exit 1 fi - echo "[cos executing] src/composer/target/debug/compose $script $name" - src/composer/target/debug/compose $script $name + if [ -z "$minlib" ]; then + echo "[cos executing] src/composer/target/debug/compose $script $name" + src/composer/target/debug/compose $script $name + local dir="system_binaries/cos_build-${name}" + else + echo "[cos executing] src/composer/target/debug/compose $script $name $minlib" + src/composer/target/debug/compose $script $name $minlib + local dir="system_binaries/cos_build_minlib-${name}" - local dir="system_binaries/cos_build-${name}" + fi tools/build_iso.sh ${dir}/cos.img } @@ -157,6 +164,7 @@ case $1 in compose ) script=$2 name=$3 + minlib=$4 compose ;; run ) diff --git a/src/Makefile b/src/Makefile index 8ccee4cba1..daa537e971 100644 --- a/src/Makefile +++ b/src/Makefile @@ -20,6 +20,13 @@ composer: component: $(MAKE) $(MAKEOPTIONS) -C components component +component_minlib: + $(MAKE) $(MAKEOPTIONS) -C components component_minlib + +component_dir: + $(MAKE) $(MAKEOPTIONS) -C components component_dir + + comps: $(info ) $(info ***********************************************) diff --git a/src/components/Makefile b/src/components/Makefile index 9aa6d489f8..ef706b407f 100644 --- a/src/components/Makefile +++ b/src/components/Makefile @@ -24,3 +24,11 @@ init: component: $(MAKE) $(MAKEOPTIONS) -C implementation component + +component_minlib: + $(MAKE) $(MAKEOPTIONS) -C lib component_minlib + $(MAKE) $(MAKEOPTIONS) -C interface component_minlib + +component_dir: + $(MAKE) $(MAKEOPTIONS) -C implementation component_dir + diff --git a/src/components/implementation/Makefile b/src/components/implementation/Makefile index 69ad4c1aa1..c925f0f30f 100644 --- a/src/components/implementation/Makefile +++ b/src/components/implementation/Makefile @@ -8,6 +8,7 @@ all: $(info ***********************************************) $(info *********[ Building Implementations ]**********) $(info ***********************************************) + $(info $(SUBDIRS)) @for dir in $(SUBDIRS) ; do \ $(MAKE) $(MAKEOPTIONS) -C $$dir ; \ done @@ -25,3 +26,7 @@ distclean: component: $(MAKE) $(MAKEOPTIONS) -C $(COMP_INTERFACE) component + +component_dir: + $(MAKE) $(MAKEOPTIONS) -C $(COMP_INTERFACE) component_dir + diff --git a/src/components/implementation/Makefile.subdir b/src/components/implementation/Makefile.subdir index 659f4c340c..554b3cab8f 100644 --- a/src/components/implementation/Makefile.subdir +++ b/src/components/implementation/Makefile.subdir @@ -77,4 +77,9 @@ clean: component: $(MAKE) -C $(COMP_NAME) component + +.PHONY: component_dir +component_dir: + $(MAKE) -C $(COMP_NAME) component_dir + -include $(SOURCE_DEPENDENCIES) diff --git a/src/components/implementation/Makefile.subsubdir b/src/components/implementation/Makefile.subsubdir index 1b179e77bd..a97d893950 100644 --- a/src/components/implementation/Makefile.subsubdir +++ b/src/components/implementation/Makefile.subsubdir @@ -79,6 +79,11 @@ COMP_DEPLIBDIRS_CLEAN=$(foreach D,$(COMP_DEPS_CLEAN),$(if $(wildcard $(INTERDIR) COMP_EXPIF_OBJS=$(foreach I,$(COMP_INTERFACES_CLEAN),$(INTERDIR)/$(I)/cosrt_s_stub.o) COMP_DEP_OBJS=$(foreach D,$(COMP_IFDEPS_CLEAN),$(INTERDIR)/$(D)/cosrt_c_stub.o) +DEP_DIRS := $(foreach obj,$(COMP_DEP_OBJS),$(dir $(obj))) +DEP_DIRS := $(foreach dir,$(DEP_DIRS),$(dir $(patsubst %/,%,$(dir)))) +DEP_OBJS := $(foreach obj,$(DEPENDENCY_LIBOBJS),$(dir $(obj))) +LIB_DIRS := $(foreach lib,$(DEPENDENCY_LIBPATH),$(subst -L,,$(lib))) + # NOTE: we're currently ignoring the *variants* library requirements, # which will break if an interface's code requires a library @@ -86,21 +91,25 @@ comp_header: $(info | Composing $(COMP_INTERFACE).$(COMP_NAME) for variable $(COMP_VARNAME) by linking with:) $(info | Exported interfaces: $(COMP_INTERFACES_CLEAN)) $(info | Interface dependencies: $(COMP_IFDEPS_CLEAN)) - $(info | Libraries: $(DEPENDENCY_LIBS) $(DEPENDENCY_LIBOBJS)) + $(info | Libraries:$(DEPENDENCY_LIBS) $(DEPENDENCY_LIBOBJS)) .PHONY: component component: clean comp_header $(COMPOBJ) - $(if $(COMP_INITARGS_FILE), $(CC) $(INCLUDE) $(CFLAGS) -c -o $(COMP_INITARGS_FILE:%.c=%.o) $(COMP_INITARGS_FILE)) + $(if $(COMP_INITARGS_FILE), $(CC) $(INCLUDE) $(CFLAGS) $(CFLAGS_MINLIB) -c -o $(COMP_INITARGS_FILE:%.c=%.o) $(COMP_INITARGS_FILE)) $(if $(COMP_TAR_FILE), cp $(COMP_TAR_FILE) $(TAR_SYMBOL_NAME)) $(if $(COMP_TAR_FILE), $(LD) $(LDFLAGS) -r -b binary $(TAR_SYMBOL_NAME) -o $(COMP_TAR_FILE).o; rm $(TAR_SYMBOL_NAME)) $(LD) $(LDFLAGS) -r -o $(COMPNAME).linked_libs_ifs.o $(COMPOBJ) $(COMP_EXPIF_OBJS) $(COMP_DEP_OBJS) $(if $(COMP_INITARGS_FILE), $(COMP_INITARGS_FILE:%.c=%.o)) $(if $(COMP_TAR_FILE), $(COMP_TAR_FILE).o) $(COMP_DEPLIBDIRS_CLEAN) $(COMP_DEPLIBS_CLEAN) $(LIB_FLAGS) $(MUSLCC) $(COMPNAME).linked_libs_ifs.o $(MUSLCFLAGS) $(LINKFLAG) -o $(COMPNAME).linked_musl.o - $(LD) $(LDFLAGS) -Ttext=$(COMP_BASEADDR) -T $(COMP_LD_SCRIPT) -o $(COMP_OUTPUT) $(COMPNAME).linked_musl.o + $(LD) $(LDFLAGS) $(LDFLAGS_MINLIB) -Ttext=$(COMP_BASEADDR) -T $(COMP_LD_SCRIPT) -o $(COMP_OUTPUT) $(COMPNAME).linked_musl.o + +.PHONY: component_dir +component_dir: + $(info $(LIB_DIRS) $(DEP_DIRS) $(DEP_OBJS)) # NOTE: make CFLAGS/CXXFLAGS/ASFLAGS argument first because it includes kernel headers/musl headers first %.o:%.c $(info | [CC] $<: Compiling) - @$(CC) $(CFLAGS) $(INCLUDE) -o $@ -c $< + @$(CC) $(CFLAGS) $(CFLAGS_MINLIB) $(INCLUDE) -o $@ -c $< %.o:%.cc $(info | [CXX] $<: Compiling) diff --git a/src/components/implementation/comp_x86_64.ld b/src/components/implementation/comp_x86_64.ld index 02ce2e6365..287a699251 100644 --- a/src/components/implementation/comp_x86_64.ld +++ b/src/components/implementation/comp_x86_64.ld @@ -11,7 +11,7 @@ SECTIONS . = SIZEOF_HEADERS; /* start the text/read-only sections */ - .text : ALIGN(4096) { *(.text*) } :text + .text : ALIGN(4096) { KEEP(*(.text.__cosrt_c_cosrtdefault)) *(.text*) } :text .rodata : { *(.rodata*) } .eh_frame : { *(.eh_frame*) } diff --git a/src/components/implementation/pong/pingpong/Makefile b/src/components/implementation/pong/pingpong/Makefile index 7b8a896832..9bf7831ec0 100644 --- a/src/components/implementation/pong/pingpong/Makefile +++ b/src/components/implementation/pong/pingpong/Makefile @@ -15,4 +15,6 @@ LIBRARY_DEPENDENCIES = component # the build to fail. The build system does not validate this # minimality; that's on you! + + include Makefile.subsubdir diff --git a/src/components/interface/Makefile b/src/components/interface/Makefile index 4e32102b91..880f1f66ad 100644 --- a/src/components/interface/Makefile +++ b/src/components/interface/Makefile @@ -8,6 +8,7 @@ all: $(info ***********************************************) $(info ************[ Building interfaces ]************) $(info ***********************************************) + $(info $(INTERFACES)) @for dir in $(INTERFACES) ; do \ $(MAKE) $(MAKEOPTIONS) -C $$dir ; \ done @@ -24,3 +25,15 @@ cp: .PHONY: init init: clean + +INTERFACE_DIRS := $(shell echo "$(MINLIB_DIRS)" | tr ' ' '\n' | grep '//interface//') +.PHONY: component_minlib +component_minlib: + $(info ***********************************************) + $(info ********[ ReBuilding MinLib Interface]*********) + $(info ***********************************************) + $(info $(INTERFACE_DIRS)) + @for dir in $(INTERFACE_DIRS) ; do \ + $(MAKE) $(MAKEOPTIONS) -C $$dir clean ; \ + $(MAKE) $(MAKEOPTIONS) -C $$dir ; \ + done \ No newline at end of file diff --git a/src/components/interface/Makefile.subsubdir b/src/components/interface/Makefile.subsubdir index cd05db91b2..bdb1fdb8b3 100644 --- a/src/components/interface/Makefile.subsubdir +++ b/src/components/interface/Makefile.subsubdir @@ -38,10 +38,11 @@ print: %.o:%.c $(info | [CC] Compiling c file $< into $@) - @$(CC) $(CFLAGS) $(DEP_INC) -c -o $(@) $< + @$(CC) $(CFLAGS) $(CFLAGS_MINLIB) $(DEP_INC) -c -o $(@) $< $(S_SSTUB_OBJ):$(SSTUB_FILE) $(info | [AS] Creating server asm stubs for $(IFNAME)) + $(info | include paths are $(DEP_INC)) @$(AS) -DCOS_SERVER_STUBS $(ASFLAGS) $(DEP_INC) -c -o $@ $^ $(C_UCAP_STUB_OBJ):$(SSTUB_FILE) diff --git a/src/components/lib/Makefile b/src/components/lib/Makefile index 7fc8aaf146..1185c9169b 100644 --- a/src/components/lib/Makefile +++ b/src/components/lib/Makefile @@ -82,6 +82,7 @@ all: $(info **************************************************) $(info **************[ Building Libraries ]**************) $(info **************************************************) + $(info $(SUBDIRS)) @for dir in $(SUBDIRS) ; do \ echo "*************************[TASK START]*************************";\ echo " [ Building $$dir ]";\ @@ -125,3 +126,18 @@ distclean: echo "*************************[TASK STOP]*************************";\ echo ""; \ done + +LIB_DIRS := $(shell echo "$(MINLIB_DIRS)" | tr ' ' '\n' | grep '/lib/') +component_minlib: + $(info **************************************************) + $(info ***********[ ReBuilding MinLib Library]***********) + $(info **************************************************) + $(info $(LIB_DIRS)) + @for dir in $(LIB_DIRS) ; do \ + echo "*************************[TASK START]*************************";\ + echo " [ Building $$dir ]";\ + $(MAKE) PLATFORM=$(PLATFORM) $(MAKEOPTIONS) -C $$dir clean || exit $$?; \ + $(MAKE) PLATFORM=$(PLATFORM) $(MAKEOPTIONS) -C $$dir || exit $$?; \ + echo "*************************[TASK STOP]*************************";\ + echo ""; \ + done \ No newline at end of file diff --git a/src/components/lib/Makefile.lib b/src/components/lib/Makefile.lib index 9bcee92c09..e25e784eeb 100644 --- a/src/components/lib/Makefile.lib +++ b/src/components/lib/Makefile.lib @@ -35,7 +35,7 @@ $(SHLIB_FILE): $(OBJS) %.o:%.c $(info | [CC] Compiling C file $< into $@) - @$(CC) $(CFLAGS) $(CINC) $(DEPENDENCY_INCPATH) -c -o $@ $< + @$(CC) $(CFLAGS) $(CFLAGS_MINLIB) $(CINC) $(DEPENDENCY_INCPATH) -c -o $@ $< # redirect error output to /dev/null so that it will not display # errors when cleaning, this does not affect gcc's error output when diff --git a/src/composer/src/build.rs b/src/composer/src/build.rs index 9f43f3d420..9f9db1bdc4 100644 --- a/src/composer/src/build.rs +++ b/src/composer/src/build.rs @@ -254,6 +254,7 @@ fn comp_gen_make_cmd( tar_file: &Option, id: &ComponentId, s: &SystemState, + is_minlib: bool, ) -> String { let c = component(&s, id); let ds = deps(&s, id); @@ -295,14 +296,31 @@ fn comp_gen_make_cmd( let compid = s.get_named().rmap().get(&c.name).unwrap(); let baseaddr = s.get_address_assignments().component_baseaddr(compid); - let cmd = format!( + let cmd = if is_minlib { + format!( + r#"make --quiet -C src COMP_INTERFACES="{}" COMP_IFDEPS="{}" COMP_LIBDEPS="" COMP_INTERFACE={} COMP_NAME={} component_dir"#, + if_exp, if_deps, &decomp[0], &decomp[1] + ) + } + else {format!( r#"make -C src COMP_INTERFACES="{}" COMP_IFDEPS="{}" COMP_LIBDEPS="" COMP_INTERFACE={} COMP_NAME={} COMP_VARNAME={} COMP_OUTPUT={} COMP_BASEADDR={:#X} {} component"#, if_exp, if_deps, &decomp[0], &decomp[1], &c.name, output_name, baseaddr, &optional_cmds - ); + ) + }; cmd } +fn comp_gen_minlib_make_cmd( + minlib_dirs: &String, + cflags_minlib: &String +) -> String { + format!( + r#"make -C src MINLIB_DIRS="{}" CFLAGS_MINLIB="{}" component_minlib"#, + minlib_dirs, cflags_minlib + ) +} + fn kern_gen_make_cmd(input_constructor: &String, kern_output: &String, _s: &SystemState) -> String { format!( r#"make -C src KERNEL_OUTPUT="{}" CONSTRUCTOR_COMP="{}" plat"#, @@ -312,12 +330,14 @@ fn kern_gen_make_cmd(input_constructor: &String, kern_output: &String, _s: &Syst pub struct DefaultBuilder { builddir: String, + minlib: bool, } impl DefaultBuilder { pub fn new() -> Self { DefaultBuilder { builddir: "/dev/null".to_string(), // must initialize, so error out if you don't + minlib: false, } } } @@ -330,15 +350,21 @@ fn compdir_check_build(comp_dir: &String) -> Result<(), String> { Ok(()) } +use std::process::Command; impl BuildState for DefaultBuilder { - fn initialize(&mut self, name: &String, _s: &SystemState) -> Result<(), String> { + fn initialize(&mut self, name: &String, _s: &SystemState, minlib: bool) -> Result<(), String> { let pwd = env::current_dir().unwrap(); - let dir = format!("{}/system_binaries/cos_build-{}", pwd.display(), name); + + let dir = if minlib { + format!("{}/system_binaries/cos_build-{}-minlib", pwd.display(), name) + } else { + format!("{}/system_binaries/cos_build-{}", pwd.display(), name) + }; reset_dir(&dir)?; self.builddir = dir; - + self.minlib = minlib; Ok(()) } @@ -378,7 +404,37 @@ impl BuildState for DefaultBuilder { let p = state.get_param_id(&id); let output_path = self.comp_obj_path(&id, &state)?; - let cmd = comp_gen_make_cmd(&output_path, p.param_prog(), p.param_fs(), &id, &state); + if self.minlib { + //get the related sub directories + let subdirs_cmd = comp_gen_make_cmd(&output_path, p.param_prog(), p.param_fs(), &id, &state, true); + let (out1, err1) = exec_pipeline(vec![subdirs_cmd.clone()]); + + if !err1.is_empty() { + return Err(format!("Error in executing pipeline for component_dir: {}", err1)); + } + + let mut cflag_minlib = "-ffunction-sections -fdata-sections"; + + //rebuild the files under the related sub directories + let output = Command::new("make") + .arg("-C") + .arg("src") + .arg(format!("MINLIB_DIRS={}", out1)) + .arg(format!("CFLAGS_MINLIB={}", cflag_minlib)) + .arg("component_minlib") + .output() + .expect("Failed to execute make command"); + + if output.status.success() { + let stdout = String::from_utf8_lossy(&output.stdout); + println!("Output: {}", stdout); + } else { + let stderr = String::from_utf8_lossy(&output.stderr); + eprintln!("Error: {}", stderr); + } + } + + let cmd = comp_gen_make_cmd(&output_path, p.param_prog(), p.param_fs(), &id, &state, false); let name = state.get_named().ids().get(id).unwrap(); println!( @@ -413,7 +469,7 @@ impl BuildState for DefaultBuilder { let binary = self.comp_obj_path(&c, &s)?; let argsfile = constructor_serialize_args(&c, &s, self)?; let tarfile = constructor_tarball_create(&c, &s, self)?; - let cmd = comp_gen_make_cmd(&binary, &argsfile, &tarfile, &c, &s); + let cmd = comp_gen_make_cmd(&binary, &argsfile, &tarfile, &c, &s, false); let name = s.get_named().ids().get(c).unwrap(); println!( diff --git a/src/composer/src/main.rs b/src/composer/src/main.rs index 0669218598..a50c6087b4 100644 --- a/src/composer/src/main.rs +++ b/src/composer/src/main.rs @@ -37,17 +37,20 @@ pub fn exec() -> Result<(), String> { let arg1 = args.next(); let arg2 = args.next(); + let arg3 = args.next(); // Optional third argument if None == arg1 || None == arg2 { return Err(format!( - "usage: {} .toml ", + "usage: {} .toml [minlib]", program_name.unwrap() )); } + let b_minlib= arg3.as_ref().map(|s| s == "minlib").unwrap_or(false); + let mut sys = SystemState::new(arg1.unwrap()); let mut build = DefaultBuilder::new(); - build.initialize(&arg2.unwrap(), &sys)?; + build.initialize(&arg2.unwrap(), &sys, b_minlib)?; sys.add_parsed(SystemSpec::transition(&sys, &mut build)?); sys.add_named(CompTotOrd::transition(&sys, &mut build)?); diff --git a/src/composer/src/passes.rs b/src/composer/src/passes.rs index f8308a32bb..21b6040d5f 100644 --- a/src/composer/src/passes.rs +++ b/src/composer/src/passes.rs @@ -128,7 +128,7 @@ impl SystemState { // that file already exists. Use unique names for files, and use the // component-namespacing of names for per-component files. pub trait BuildState { - fn initialize(&mut self, name: &String, s: &SystemState) -> Result<(), String>; // must be called *before* the following functions + fn initialize(&mut self, name: &String, s: &SystemState, minlib: bool) -> Result<(), String>; // must be called *before* the following functions fn file_path(&self, file: &String) -> Result; // create a path in the build directory for a file fn comp_dir_path(&self, c: &ComponentId, state: &SystemState) -> Result; // the component's object fn comp_file_path( diff --git a/src/composer/src/resources.rs b/src/composer/src/resources.rs index ba3d025145..230de46420 100644 --- a/src/composer/src/resources.rs +++ b/src/composer/src/resources.rs @@ -15,7 +15,7 @@ enum CapRes { Comp(ComponentId), } -const BOOT_CAPTBL_FREE: u32 = 60; +const BOOT_CAPTBL_FREE: u32 = 52; // The equivalent of the C __captbl_cap2sz(c) fn cap_sz(cap: &CapRes) -> u32 { diff --git a/src/composer/src/syshelpers.rs b/src/composer/src/syshelpers.rs index e9c7652ef1..4f8e3ea304 100644 --- a/src/composer/src/syshelpers.rs +++ b/src/composer/src/syshelpers.rs @@ -6,9 +6,11 @@ use std::fs; pub fn exec_pipeline(progs: Vec) -> (String, String) { let output = progs.iter() .fold(None, - |upstream: Option, cmd| match upstream { + |upstream: Option, cmd| { println!("Executing command: {}", cmd); // This line added to print the command + match upstream { None => Some(Pipe::new(cmd)), // initial command Some(up) => Some(up.next(cmd)) // piped commands + } }) .unwrap_or_else(|| Pipe::new("cat /dev/null")) .output()