procyberian/emacs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Do not allocate huge temporary memory areas and objects while encoding

Browse source 
for file I/O, thus reducing an enormous memory usage for large buffers.
See http://lists.gnu.org/archive/html/emacs-devel/2013-10/msg00180.html.
* coding.h (struct coding_system): New member raw_destination.
* coding.c (setup_coding_system): Initialize it to zero.
(encode_coding_object): If raw_destination is set, do not create
dst_object.  Add comment.
* fileio.c (toplevel): New constant E_WRITE_MAX.
(e_write): Do not encode more than E_WRITE_MAX characters per one loop
iteration.  Use raw_destination if E_WRITE_MAX characters is encoded.
This commit is contained in:
Dmitry Antipov 2013-10-08 10:40:09 +04:00
parent b7d5bd823c
commit f849808191
4 changed files with 57 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
src/ChangeLog
View file

@ -1,3 +1,16 @@
2013-10-08 Dmitry Antipov <dmantipov@yandex.ru>
Do not allocate huge temporary memory areas and objects while encoding
for file I/O, thus reducing an enormous memory usage for large buffers.
See http://lists.gnu.org/archive/html/emacs-devel/2013-10/msg00180.html.
* coding.h (struct coding_system): New member raw_destination.
* coding.c (setup_coding_system): Initialize it to zero.
(encode_coding_object): If raw_destination is set, do not create
dst_object. Add comment.
* fileio.c (toplevel): New constant E_WRITE_MAX.
(e_write): Do not encode more than E_WRITE_MAX characters per one loop
iteration. Use raw_destination if E_WRITE_MAX characters is encoded.
2013-10-08 Jan Djärv <jan.h.d@swipnet.se>
* nsterm.m (windowDidExitFullScreen:):

6
src/coding.c
View file

@ -5761,6 +5761,7 @@ setup_coding_system (Lisp_Object coding_system, struct coding_system *coding)
coding->safe_charsets = SDATA (val);
coding->default_char = XINT (CODING_ATTR_DEFAULT_CHAR (attrs));
coding->carryover_bytes = 0;
coding->raw_destination = 0;
coding_type = CODING_ATTR_TYPE (attrs);
if (EQ (coding_type, Qundecided))
@ -8352,6 +8353,11 @@ encode_coding_object (struct coding_system *coding,
{
if (BUFFERP (coding->dst_object))
coding->dst_object = Fbuffer_string ();
else if (coding->raw_destination)
/* This is used to avoid creating huge Lisp string.
NOTE: caller who sets `raw_destination' is also
responsible for freeing `destination' buffer. */
coding->dst_object = Qnil;
else
{
coding->dst_object

4
src/coding.h
View file

@ -512,6 +512,10 @@ struct coding_system
`charbuf', but at `src_object'. */
unsigned chars_at_source : 1;
/* Nonzero if the result of conversion is in `destination'
buffer rather than in `dst_object'. */
unsigned raw_destination : 1;
/* Set to 1 if charbuf contains an annotation. */
unsigned annotated : 1;

42
src/fileio.c
View file

@ -5263,6 +5263,10 @@ a_write (int desc, Lisp_Object string, ptrdiff_t pos,
return 1;
}
/* Maximum number of characters that the next
function encodes per one loop iteration. */
enum { E_WRITE_MAX = 8 * 1024 * 1024 };
/* Write text in the range START and END into descriptor DESC,
encoding them with coding system CODING. If STRING is nil, START
@ -5289,9 +5293,16 @@ e_write (int desc, Lisp_Object string, ptrdiff_t start, ptrdiff_t end,
coding->src_multibyte = SCHARS (string) < SBYTES (string);
if (CODING_REQUIRE_ENCODING (coding))
{
encode_coding_object (coding, string,
start, string_char_to_byte (string, start),
end, string_char_to_byte (string, end), Qt);
ptrdiff_t nchars = min (end - start, E_WRITE_MAX);
/* Avoid creating huge Lisp string in encode_coding_object. */
if (nchars == E_WRITE_MAX)
coding->raw_destination = 1;
encode_coding_object
(coding, string, start, string_char_to_byte (string, start),
start + nchars, string_char_to_byte (string, start + nchars),
Qt);
}
else
{
@ -5308,8 +5319,15 @@ e_write (int desc, Lisp_Object string, ptrdiff_t start, ptrdiff_t end,
coding->src_multibyte = (end - start) < (end_byte - start_byte);
if (CODING_REQUIRE_ENCODING (coding))
{
encode_coding_object (coding, Fcurrent_buffer (),
start, start_byte, end, end_byte, Qt);
ptrdiff_t nchars = min (end - start, E_WRITE_MAX);
/* Likewise. */
if (nchars == E_WRITE_MAX)
coding->raw_destination = 1;
encode_coding_object
(coding, Fcurrent_buffer (), start, start_byte,
start + nchars, CHAR_TO_BYTE (start + nchars), Qt);
}
else
{
@ -5330,11 +5348,19 @@ e_write (int desc, Lisp_Object string, ptrdiff_t start, ptrdiff_t end,
if (coding->produced > 0)
{
char *buf = (STRINGP (coding->dst_object)
? SSDATA (coding->dst_object)
: (char *) BYTE_POS_ADDR (coding->dst_pos_byte));
char *buf = (coding->raw_destination ? (char *) coding->destination
: (STRINGP (coding->dst_object)
? SSDATA (coding->dst_object)
: (char *) BYTE_POS_ADDR (coding->dst_pos_byte)));
coding->produced -= emacs_write_sig (desc, buf, coding->produced);
if (coding->raw_destination)
{
/* We're responsible for freeing this, see
encode_coding_object to check why. */
xfree (coding->destination);
coding->raw_destination = 0;
}
if (coding->produced)
return 0;
}