diff --git a/libswscale/swscale.c b/libswscale/swscale.c
index 37c7cf60dd..2db40a6807 100644
--- a/libswscale/swscale.c
+++ b/libswscale/swscale.c
@@ -871,7 +871,6 @@ int attribute_align_arg sws_scale(struct SwsContext *c,
     int i, ret;
     const uint8_t *src2[4];
     uint8_t *dst2[4];
-    uint8_t *rgb0_tmp = NULL;
     int macro_height = isBayer(c->srcFormat) ? 2 : (1 << c->chrSrcVSubSample);
     // copy strides, so they can safely be modified
     int srcStride2[4];
@@ -928,11 +927,14 @@ int attribute_align_arg sws_scale(struct SwsContext *c,
     if (c->src0Alpha && !c->dst0Alpha && isALPHA(c->dstFormat)) {
         uint8_t *base;
         int x,y;
-        rgb0_tmp = av_malloc(FFABS(srcStride[0]) * srcSliceH + 32);
-        if (!rgb0_tmp)
+
+        av_fast_malloc(&c->rgb0_scratch, &c->rgb0_scratch_allocated,
+                       FFABS(srcStride[0]) * srcSliceH + 32);
+        if (!c->rgb0_scratch)
             return AVERROR(ENOMEM);
 
-        base = srcStride[0] < 0 ? rgb0_tmp - srcStride[0] * (srcSliceH-1) : rgb0_tmp;
+        base = srcStride[0] < 0 ? c->rgb0_scratch - srcStride[0] * (srcSliceH-1) :
+                                  c->rgb0_scratch;
         for (y=0; y<srcSliceH; y++){
             memcpy(base + srcStride[0]*y, src2[0] + srcStride[0]*y, 4*c->srcW);
             for (x=c->src0Alpha-1; x<4*c->srcW; x+=4) {
@@ -944,11 +946,14 @@ int attribute_align_arg sws_scale(struct SwsContext *c,
 
     if (c->srcXYZ && !(c->dstXYZ && c->srcW==c->dstW && c->srcH==c->dstH)) {
         uint8_t *base;
-        rgb0_tmp = av_malloc(FFABS(srcStride[0]) * srcSliceH + 32);
-        if (!rgb0_tmp)
+
+        av_fast_malloc(&c->xyz_scratch, &c->xyz_scratch_allocated,
+                       FFABS(srcStride[0]) * srcSliceH + 32);
+        if (!c->xyz_scratch)
             return AVERROR(ENOMEM);
 
-        base = srcStride[0] < 0 ? rgb0_tmp - srcStride[0] * (srcSliceH-1) : rgb0_tmp;
+        base = srcStride[0] < 0 ? c->xyz_scratch - srcStride[0] * (srcSliceH-1) :
+                                  c->xyz_scratch;
 
         xyz12Torgb48(c, (uint16_t*)base, (const uint16_t*)src2[0], srcStride[0]/2, srcSliceH);
         src2[0] = base;
@@ -996,6 +1001,5 @@ int attribute_align_arg sws_scale(struct SwsContext *c,
         rgb48Toxyz12(c, dst16, dst16, dstStride2[0]/2, ret);
     }
 
-    av_free(rgb0_tmp);
     return ret;
 }
diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h
index a1de95cee0..9304a5ef42 100644
--- a/libswscale/swscale_internal.h
+++ b/libswscale/swscale_internal.h
@@ -626,6 +626,18 @@ typedef struct SwsContext {
     SwsDither dither;
 
     SwsAlphaBlend alphablend;
+
+    // scratch buffer for converting packed rgb0 sources
+    // filled with a copy of the input frame + fully opaque alpha,
+    // then passed as input to further conversion
+    uint8_t     *rgb0_scratch;
+    unsigned int rgb0_scratch_allocated;
+
+    // scratch buffer for converting XYZ sources
+    // filled with the input converted to rgb48
+    // then passed as input to further conversion
+    uint8_t     *xyz_scratch;
+    unsigned int xyz_scratch_allocated;
 } SwsContext;
 //FIXME check init (where 0)
 
diff --git a/libswscale/utils.c b/libswscale/utils.c
index d673209d95..974e2d5179 100644
--- a/libswscale/utils.c
+++ b/libswscale/utils.c
@@ -2295,6 +2295,9 @@ void sws_freeContext(SwsContext *c)
     av_freep(&c->gamma);
     av_freep(&c->inv_gamma);
 
+    av_freep(&c->rgb0_scratch);
+    av_freep(&c->xyz_scratch);
+
     ff_free_filters(c);
 
     av_free(c);