3 # Copyright (C) 2013 Intra2net AG
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU Lesser General Public License as published
7 # by the Free Software Foundation; either version 3 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU Lesser General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see
17 # <http://www.gnu.org/licenses/lgpl-3.0.html>
24 def split_file(separator, prefix, input_file, new_file_func=None):
26 splits a file when it finds a regexp, including the regexp in the begining
32 sep_len = len(separator)
34 raise Exception("empty separator")
36 if new_file_func is None:
37 new_file_func = lambda prefix, i: open(prefix + str(i), 'wb')
39 output = new_file_func(prefix, i)
41 # buffered search. we try not to have the while input file in memory, as
43 with open(input_file, 'rb') as f:
45 buf += f.read(BUFSIZE)
49 # split using the separator
50 while separator in buf:
51 idx = buf.index(separator)
54 output.write(buf[0:idx])
57 output = new_file_func(prefix, i)
58 output.write(buf[idx:idx + sep_len])
60 output.write(buf[0:sep_len])
62 buf = buf[idx + sep_len:]
64 # corner case: separator is between this buf and next one. In this
65 # case, we write to current output everything before that and
67 if len(buf) > sep_len and separator[0] in buf[-sep_len:]:
68 output.write(buf[:-sep_len])
72 # else: continue writing to the current output and iterate
78 def chunk_file(input_file, output_file, from_pos, to_pos):
79 ifl = open(input_file, 'rb')
80 ofl = open(output_file, 'wb')
83 ofl.write(ifl.read(to_pos-from_pos))
87 if __name__ == "__main__":
88 parser = argparse.ArgumentParser()
90 parser.add_argument("-s", "--separator", help="string for the separator")
91 parser.add_argument("-p", "--prefix", help="prefix for split files")
92 parser.add_argument("input_file", help="input file")
93 parser.add_argument("-f", "--from-pos", type=int, default=-1)
94 parser.add_argument("-t", "--to-pos", type=int, default=-1)
95 parser.add_argument("-o", "--output", default=None)
97 args = parser.parse_args()
98 if args.from_pos > -1 and args.to_pos > -1:
99 chunk_file(input_file=args.input_file, output_file=args.output,
100 from_pos=args.from_pos, to_pos=args.to_pos)
102 split_file(separator=args.separator.encode('UTF-8', errors="surrogateescape"),
103 prefix=args.prefix, input_file=args.input_file)