Fix CVE-2016-7613 (Integer overflow in opj_pi_create_decode allowing execution
of arbitrary code):

https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-7163
https://github.com/uclouvain/openjpeg/issues/826
http://seclists.org/oss-sec/2016/q3/442

Copied from upstream repository:

https://github.com/uclouvain/openjpeg/commit/c16bc057ba3f125051c9966cf1f5b68a05681de4
https://github.com/uclouvain/openjpeg/commit/ef01f18dfc6780b776d0674ed3e7415c6ef54d24

From c16bc057ba3f125051c9966cf1f5b68a05681de4 Mon Sep 17 00:00:00 2001
From: trylab <trylab@users.noreply.github.com>
Date: Tue, 6 Sep 2016 13:55:49 +0800
Subject: [PATCH] Fix an integer overflow issue (#809)

Prevent an integer overflow issue in function opj_pi_create_decode of
pi.c.
---
 src/lib/openjp2/pi.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/lib/openjp2/pi.c b/src/lib/openjp2/pi.c
index cffad66..36e2ff0 100644
--- a/src/lib/openjp2/pi.c
+++ b/src/lib/openjp2/pi.c
@@ -1237,7 +1237,13 @@ opj_pi_iterator_t *opj_pi_create_decode(opj_image_t *p_image,
 	l_current_pi = l_pi;
 
 	/* memory allocation for include */
-	l_current_pi->include = (OPJ_INT16*) opj_calloc((l_tcp->numlayers +1) * l_step_l, sizeof(OPJ_INT16));
+	/* prevent an integer overflow issue */
+	l_current_pi->include = 00;
+	if (l_step_l <= (SIZE_MAX / (l_tcp->numlayers + 1U)))
+	{
+		l_current_pi->include = (OPJ_INT16*) opj_calloc((l_tcp->numlayers +1) * l_step_l, sizeof(OPJ_INT16));
+	}
+
 	if
 		(!l_current_pi->include)
 	{
-- 
2.10.0

From ef01f18dfc6780b776d0674ed3e7415c6ef54d24 Mon Sep 17 00:00:00 2001
From: Matthieu Darbois <mayeut@users.noreply.github.com>
Date: Thu, 8 Sep 2016 07:34:46 +0200
Subject: [PATCH] Cast to size_t before multiplication

Need to cast to size_t before multiplication otherwise overflow check is useless.
---
 src/lib/openjp2/pi.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib/openjp2/pi.c b/src/lib/openjp2/pi.c
index 36e2ff0..809b33d 100644
--- a/src/lib/openjp2/pi.c
+++ b/src/lib/openjp2/pi.c
@@ -1241,7 +1241,7 @@ opj_pi_iterator_t *opj_pi_create_decode(opj_image_t *p_image,
 	l_current_pi->include = 00;
 	if (l_step_l <= (SIZE_MAX / (l_tcp->numlayers + 1U)))
 	{
-		l_current_pi->include = (OPJ_INT16*) opj_calloc((l_tcp->numlayers +1) * l_step_l, sizeof(OPJ_INT16));
+		l_current_pi->include = (OPJ_INT16*) opj_calloc((size_t)(l_tcp->numlayers + 1U) * l_step_l, sizeof(OPJ_INT16));
 	}
 
 	if
-- 
2.10.0

