Skip to content

Commit

Permalink
mkrufky#5 Addedsupport of oqutput sequence in subdir + added output f…
Browse files Browse the repository at this point in the history 
…iles sequence size limit
  • Loading branch information
@pashamesh
pashamesh committed Aug 10, 2015
1 parent f524adc commit e5b27c7
Show file tree
Hide file tree
Showing 3 changed files with 140 additions and 45 deletions.
43 changes: 28 additions & 15 deletions dvbtee/dvbtee.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,8 @@ void usage(bool help, char *myname)
"-I\t\trequest a service and its associated PES streams by its service id\n "
"-E\t\tenable EPG scan, optional arg to limit the number of EITs to parse\n "
"-o\t\toutput filtered data, optional arg is a filename / URI, ie udp://127.0.0.1:1234\n "
"-r[limit]\tRotate output file when size limit reached.\n\t\tUsed together with -ofile... option \n\t\tLimit specified in bytes, default is 1048576 (1 MB).\n "
"-r[limit]\tRotate output file when size limit reached.\n\t\tUse together with -ofile... option \n\t\tLimit specified in bytes, default is 1048576 (1 MB).\n "
"-l[limit]\tSet output files sequence size limit\n\t\tWorks only together with -r option\n\t\tLimit specified in bytes, default is unlimited.\n "
"-O\t\toutput options: (or-able) 1 = PAT/PMT, 2 = PES, 4 = PSIP\n "
"-H\t\tuse a HdHomeRun device, optional arg to specify the device string\n "
"-d\t\tdebug level\n "
Expand Down Expand Up @@ -364,8 +365,8 @@ int main(int argc, char **argv)
unsigned int wait_event = 0;
int eit_limit = -1;

bool b_output_file_rotate = false;
unsigned long int output_file_size_limit = 1048576; // 1MB
unsigned long int output_file_size_limit = 0;
unsigned long int output_fseq_size_limit = 0;

tune *tuner = NULL;

Expand All @@ -389,7 +390,7 @@ int main(int argc, char **argv)
char hdhrname[256];
memset(&hdhrname, 0, sizeof(hdhrname));

while ((opt = getopt(argc, argv, "a:A:bc:C:f:F:t:T:i:I:s::S::E::o::O:r::d::H::h?")) != -1) {
while ((opt = getopt(argc, argv, "a:A:bc:C:f:F:t:T:i:I:s::S::E::o::O:r::l::d::H::h?")) != -1) {
switch (opt) {
case 'a': /* adapter */
#ifdef USE_LINUXTV
Expand Down Expand Up @@ -478,14 +479,27 @@ int main(int argc, char **argv)
break;
}

b_output_file_rotate = true;
output_file_size_limit = 1048576; // 1MB default value

if (optarg) {
//TODO: detect human readable values
// (1M = 1 MB, 10K = 10KB, ...)
output_file_size_limit = strtoul(optarg, NULL, 0);
}

break;
case 'l': /* limit size of output files sequence */

This comment has been minimized.

Sign in to view

Copy link
@mkrufky

mkrufky Aug 11, 2015

what if the user wants to output a single file and cap it's size limit? perhaps we can make this feature work if the user specifies -ofilename and -lsizelimit but does not specify -r

This comment has been minimized.

Sign in to view

Copy link
@pashamesh

pashamesh Aug 12, 2015

Author Owner

Yes. It can be implemented in future
if (output_file_size_limit == 0) {
break;
}

This comment has been minimized.

Sign in to view

Copy link
@mkrufky

mkrufky Aug 11, 2015

trailing whitespace... and just skip one line, not two

if (optarg) {
//TODO: detect human readable values
// (1M = 1 MB, 10K = 10KB, ...)
output_fseq_size_limit = strtoul(optarg, NULL, 0);
}

break;
case 'O': /* output options */
out_opt = (enum output_options)strtoul(optarg, NULL, 0);
Expand Down Expand Up @@ -594,26 +608,25 @@ int main(int argc, char **argv)
}

if (b_output_file) {
fprintf(stderr, "output_file_size_limit: %lu, output_fseq_size_limit: %lu\n", output_file_size_limit, output_fseq_size_limit);


This comment has been minimized.

Sign in to view

Copy link
@mkrufky

mkrufky Aug 11, 2015

trailing whitespace... and just skip one line, not two
if (b_READ_TUNER) { // FIXME
for (
map_tuners::const_iterator iter = context.tuners.begin();
iter != context.tuners.end();
++iter
) {
if (b_output_file_rotate) {
iter->second->feeder.parser.out.rotate(
output_file_size_limit
);
}
iter->second->feeder.parser.out.rotate(
output_file_size_limit, output_fseq_size_limit
);

iter->second->feeder.parser.add_output(outfilename);
}
} else {
if (b_output_file_rotate) {
context._file_feeder.parser.out.rotate(
output_file_size_limit
);
}
context._file_feeder.parser.out.rotate(
output_file_size_limit, output_fseq_size_limit
);
context._file_feeder.parser.add_output(outfilename);
}
} // if (b_output_file)
Expand Down
125 changes: 99 additions & 26 deletions libdvbtee/output.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
#include <string>
#include <sstream>
#include <netdb.h>
#include <dirent.h>

#include "output.h"
#include "log.h"
Expand Down Expand Up @@ -220,6 +221,7 @@ output_stream::output_stream()
, sock(-1)
, mimetype(MIMETYPE_OCTET_STREAM)
, target_file_size_limit(0)
, target_fseq_size_limit(0)
, ringbuffer()
, stream_method(OUTPUT_STREAM_UDP)
, count_in(0)
Expand Down Expand Up @@ -506,7 +508,7 @@ int output_stream::stream(uint8_t* p_data, int size)
break;
case OUTPUT_STREAM_FILE:
//dprintf("(sock: %d, size: %d, filesize: %lu, limit: %lu)", sock, size, get_file_size(sock), target_file_size_limit);

if (

This comment has been minimized.

Sign in to view

Copy link
@mkrufky

mkrufky Aug 11, 2015

adding trailing whitespace again
target_file_size_limit > 0
&& ((unsigned)get_file_size(sock) + size) >= target_file_size_limit
Expand Down Expand Up @@ -576,27 +578,32 @@ int output_stream::change_file()

new_name = target_file_name;

if (target_file_size_limit > 0){
if (detect_printf_seq(target_file_name)) {
dprintf("sequence detected");

char buff[100];
snprintf(buff, sizeof(buff), target_file_name, target_file_name_index);
new_name = buff;
} else {
dprintf("sequence not detected");

std::stringstream tmp_stream;
tmp_stream << target_file_name << "_" << target_file_name_index;
tmp_stream >> new_name;
if (target_file_size_limit > 0) {
dprintf("rotate detected");
struct stat s;

//TODO: check only once
if (stat(target_file_name, &s) != 0) {
dprintf("Output directory not exists. Creating ...");
if (mkdir(target_file_name, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != 0) {
perror("Can't create output directory");
return -1;
}
}

std::stringstream tmp_stream;
tmp_stream << target_file_name << "/";
tmp_stream << target_file_name_index << ".ts";
tmp_stream >> new_name;
}

//dprintf("sock: %d, old: %s, new: %s", sock, target_file, new_name.c_str());

if (sock >= 0) {
close(sock);
//sock = -1;
sock = -1;

if (target_file_size_limit > 0) {
cleanup_target_dir();
}
}

dprintf("opening file %s...", new_name.c_str());
Expand Down Expand Up @@ -668,6 +675,73 @@ long output_stream::get_file_size(int fd)
return rc == 0 ? stat_buf.st_size : -1;
}

int filter_dir(const struct dirent *d)

This comment has been minimized.

Sign in to view

Copy link
@mkrufky

mkrufky Aug 11, 2015

i don't think you exported this function, can we make it static?

This comment has been minimized.

Sign in to view

Copy link
@pashamesh

pashamesh Aug 12, 2015

Author Owner

ok. will do.
{
if (
strcmp(d->d_name, ".") == 0
|| strcmp(d->d_name, "..") == 0
) {
return 0;
}

return 1;
}

void output_stream::cleanup_target_dir()

This comment has been minimized.

Sign in to view

Copy link
@mkrufky

mkrufky Aug 11, 2015

this is probably fine, but I don't really know what it's used for... could you just explain it a bit? maybe add some more comments

This comment has been minimized.

Sign in to view

Copy link
@pashamesh

pashamesh Aug 12, 2015

Author Owner

it's filter function for scandir int (*filter)(const struct dirent *).
See http://manpages.courier-mta.org/htmlman3/scandir.3.html
{
if (target_fseq_size_limit == 0) {
return;
}

struct dirent **namelist;
int n, i;

struct stat s;
unsigned long total_size = 0;
std::string path;

n = scandir(target_file_name, &namelist, filter_dir, versionsort);

if (n < 0) {
perror("scandir");
return;
};

// Calculating dir total ...
for(i = 0;i < n; i++) {
path.assign(target_file_name);
path.append("/");
path.append(namelist[i]->d_name);

if (stat(path.c_str(), &s) != 0) {
continue;
}

total_size += s.st_size;
}

//Removing files ...
i = 0;
while(i < n && total_size >= target_fseq_size_limit) {
path.assign(target_file_name);
path.append("/");
path.append(namelist[i]->d_name);

i++;

if (stat(path.c_str(), &s) != 0 || remove(path.c_str()) != 0) {
continue;
}


total_size -= s.st_size;

dprintf("Removed %s, total: %lu", path.c_str(), total_size);
}

free(namelist);
}

int output_stream::add(void* priv, stream_callback callback, map_pidtype &pids)
{
stream_cb = callback;
Expand Down Expand Up @@ -763,7 +837,7 @@ int output_stream::add(char* target, map_pidtype &pids)
if (b_file) {
target_file_name = ip;
target_file_name_index = 0;

This comment has been minimized.

Sign in to view

Copy link
@mkrufky

mkrufky Aug 11, 2015

added trailing whitespace...
if (change_file() < 0) {
return -1;
}
Expand Down Expand Up @@ -836,6 +910,7 @@ output::output()
, num_targets(0)
, options(OUTPUT_NONE)
, file_size_limit(0)
, fseq_size_limit(0)
, count_in(0)
, count_out(0)
{
Expand Down Expand Up @@ -1233,25 +1308,23 @@ int output::__add(char* target, map_pidtype &pids)
{
int search_id = search(target);
if (search_id >= 0) {
dprintf("target already exists #%d: %s", search_id, target);
return search_id;
dprintf("target already exists #%d: %s", search_id, target);
return search_id;

This comment has been minimized.

Sign in to view

Copy link
@mkrufky

mkrufky Aug 11, 2015

wondering why this whitespace was changed here too... you didnt change any code here :-P
}
int target_id = num_targets;

dprintf("(%d->%s)", target_id, target);

This comment has been minimized.

Sign in to view

Copy link
@mkrufky

mkrufky Aug 11, 2015

trailing whitespace...
/* push data into output buffer */
output_streams[target_id].rotate(file_size_limit, fseq_size_limit);

int ret = output_streams[target_id].add(target, pids);


if (ret == 0) {
num_targets++;

dprintf("Set file size limit to %lu", file_size_limit);

output_streams[target_id].rotate(file_size_limit);
} else
dprintf("failed to add target #%d: %s", target_id, target);
dprintf("failed to add target #%d: %s", target_id, target);

dprintf("~(%d->%s)", target_id, target);

Expand Down
17 changes: 13 additions & 4 deletions libdvbtee/output.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,14 @@ class output_stream
void stop();
inline void stop_after_drain() { if (drain()) stop(); }
int change_file();
void rotate(unsigned long int limit) { target_file_size_limit = limit; }

void rotate(unsigned long int file, unsigned long int fseq) {
target_file_size_limit = file;
target_fseq_size_limit = fseq;
}

bool detect_printf_seq(const std::string&);
void cleanup_target_dir();
void close_file();

bool push(uint8_t*, int);
Expand Down Expand Up @@ -122,7 +128,7 @@ class output_stream

char* target_file_name;
unsigned int target_file_name_index;
unsigned long int target_file_size_limit;
unsigned long int target_file_size_limit, target_fseq_size_limit;

rbuf ringbuffer;

Expand Down Expand Up @@ -197,7 +203,10 @@ class output : public socket_listen_iface
int add_http_server(int);

void set_options(enum output_options opt = OUTPUT_NONE) { options = opt; }
void rotate(unsigned long int limit) { file_size_limit = limit; }
void rotate(unsigned long int file, unsigned long int fseq) {
file_size_limit = file;
fseq_size_limit = fseq;
}

bool check();

Expand Down Expand Up @@ -229,7 +238,7 @@ class output : public socket_listen_iface

enum output_options options;

unsigned long int file_size_limit;
unsigned long int file_size_limit, fseq_size_limit;

This comment has been minimized.

Sign in to view

Copy link
@mkrufky

mkrufky Aug 11, 2015

as I mentioned in the previous patch, I would prefer not to have these as properties of the output class. let's only pass them though the output class and allow them to be configured differently in each output_stream class

This comment has been minimized.

Sign in to view

Copy link
@pashamesh

pashamesh Aug 12, 2015

Author Owner

Ok. Will try to do this w/o store in output.

unsigned long int count_in, count_out;

Expand Down

0 comments on commit e5b27c7

Please sign in to comment.