Fix CVE-2017-17785:

https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-17785
https://bugzilla.gnome.org/show_bug.cgi?id=739133

Patch copied from upstream source repository:

https://git.gnome.org/browse/gimp/commit/?id=1882bac996a20ab5c15c42b0c5e8f49033a1af54

From 1882bac996a20ab5c15c42b0c5e8f49033a1af54 Mon Sep 17 00:00:00 2001
From: Tobias Stoeckmann <tobias@stoeckmann.org>
Date: Sun, 29 Oct 2017 15:19:41 +0100
Subject: [PATCH] Bug 739133 - (CVE-2017-17785) Heap overflow while parsing FLI
 files.

It is possible to trigger a heap overflow while parsing FLI files. The
RLE decoder is vulnerable to out of boundary writes due to lack of
boundary checks.

The variable "framebuf" points to a memory area which was allocated
with fli_header->width * fli_header->height bytes. The RLE decoder
therefore must never write beyond that limit.

If an illegal frame is detected, the parser won't stop, which means
that the next valid sequence is properly parsed again. This should
allow GIMP to parse FLI files as good as possible even if they are
broken by an attacker or by accident.

While at it, I changed the variable xc to be of type size_t, because
the multiplication of width and height could overflow a 16 bit type.

Signed-off-by: Tobias Stoeckmann <tobias@stoeckmann.org>
(cherry picked from commit edb251a7ef1602d20a5afcbf23f24afb163de63b)
---
 plug-ins/file-fli/fli.c | 50 ++++++++++++++++++++++++++++++++++---------------
 1 file changed, 35 insertions(+), 15 deletions(-)

diff --git a/plug-ins/file-fli/fli.c b/plug-ins/file-fli/fli.c
index 313efeb977..ffb651e2af 100644
--- a/plug-ins/file-fli/fli.c
+++ b/plug-ins/file-fli/fli.c
@@ -25,6 +25,8 @@
 
 #include "config.h"
 
+#include <glib/gstdio.h>
+
 #include <string.h>
 #include <stdio.h>
 
@@ -461,23 +463,27 @@ void fli_read_brun(FILE *f, s_fli_header *fli_header, unsigned char *framebuf)
 	unsigned short yc;
 	unsigned char *pos;
 	for (yc=0; yc < fli_header->height; yc++) {
-		unsigned short xc, pc, pcnt;
+		unsigned short pc, pcnt;
+		size_t n, xc;
 		pc=fli_read_char(f);
 		xc=0;
 		pos=framebuf+(fli_header->width * yc);
+		n=(size_t)fli_header->width * (fli_header->height-yc);
 		for (pcnt=pc; pcnt>0; pcnt--) {
 			unsigned short ps;
 			ps=fli_read_char(f);
 			if (ps & 0x80) {
 				unsigned short len;
-				for (len=-(signed char)ps; len>0; len--) {
+				for (len=-(signed char)ps; len>0 && xc<n; len--) {
 					pos[xc++]=fli_read_char(f);
 				}
 			} else {
 				unsigned char val;
+				size_t len;
+				len=MIN(n-xc,ps);
 				val=fli_read_char(f);
-				memset(&(pos[xc]), val, ps);
-				xc+=ps;
+				memset(&(pos[xc]), val, len);
+				xc+=len;
 			}
 		}
 	}
@@ -564,25 +570,34 @@ void fli_read_lc(FILE *f, s_fli_header *fli_header, unsigned char *old_framebuf,
 	memcpy(framebuf, old_framebuf, fli_header->width * fli_header->height);
 	firstline = fli_read_short(f);
 	numline = fli_read_short(f);
+	if (numline > fli_header->height || fli_header->height-numline < firstline)
+		return;
+
 	for (yc=0; yc < numline; yc++) {
-		unsigned short xc, pc, pcnt;
+		unsigned short pc, pcnt;
+		size_t n, xc;
 		pc=fli_read_char(f);
 		xc=0;
 		pos=framebuf+(fli_header->width * (firstline+yc));
+		n=(size_t)fli_header->width * (fli_header->height-firstline-yc);
 		for (pcnt=pc; pcnt>0; pcnt--) {
 			unsigned short ps,skip;
 			skip=fli_read_char(f);
 			ps=fli_read_char(f);
-			xc+=skip;
+			xc+=MIN(n-xc,skip);
 			if (ps & 0x80) {
 				unsigned char val;
+				size_t len;
 				ps=-(signed char)ps;
 				val=fli_read_char(f);
-				memset(&(pos[xc]), val, ps);
-				xc+=ps;
+				len=MIN(n-xc,ps);
+				memset(&(pos[xc]), val, len);
+				xc+=len;
 			} else {
-				fread(&(pos[xc]), ps, 1, f);
-				xc+=ps;
+				size_t len;
+				len=MIN(n-xc,ps);
+				fread(&(pos[xc]), len, 1, f);
+				xc+=len;
 			}
 		}
 	}
@@ -689,7 +704,8 @@ void fli_read_lc_2(FILE *f, s_fli_header *fli_header, unsigned char *old_framebu
 	yc=0;
 	numline = fli_read_short(f);
 	for (lc=0; lc < numline; lc++) {
-		unsigned short xc, pc, pcnt, lpf, lpn;
+		unsigned short pc, pcnt, lpf, lpn;
+		size_t n, xc;
 		pc=fli_read_short(f);
 		lpf=0; lpn=0;
 		while (pc & 0x8000) {
@@ -700,26 +716,30 @@ void fli_read_lc_2(FILE *f, s_fli_header *fli_header, unsigned char *old_framebu
 			}
 			pc=fli_read_short(f);
 		}
+		yc=MIN(yc, fli_header->height);
 		xc=0;
 		pos=framebuf+(fli_header->width * yc);
+		n=(size_t)fli_header->width * (fli_header->height-yc);
 		for (pcnt=pc; pcnt>0; pcnt--) {
 			unsigned short ps,skip;
 			skip=fli_read_char(f);
 			ps=fli_read_char(f);
-			xc+=skip;
+			xc+=MIN(n-xc,skip);
 			if (ps & 0x80) {
 				unsigned char v1,v2;
 				ps=-(signed char)ps;
 				v1=fli_read_char(f);
 				v2=fli_read_char(f);
-				while (ps>0) {
+				while (ps>0 && xc+1<n) {
 					pos[xc++]=v1;
 					pos[xc++]=v2;
 					ps--;
 				}
 			} else {
-				fread(&(pos[xc]), ps, 2, f);
-				xc+=ps << 1;
+				size_t len;
+				len=MIN((n-xc)/2,ps);
+				fread(&(pos[xc]), len, 2, f);
+				xc+=len << 1;
 			}
 		}
 		if (lpf) pos[xc]=lpn;
-- 
2.15.1

