diff --git a/libavcodec/cbs_av1.c b/libavcodec/cbs_av1.c
index d132f36f93..bbe4461130 100644
--- a/libavcodec/cbs_av1.c
+++ b/libavcodec/cbs_av1.c
@@ -939,6 +939,8 @@ static int cbs_av1_read_unit(CodedBitstreamContext *ctx,
         priv->spatial_id  = 0;
     }
 
+    priv->ref = (AV1ReferenceFrameState *)&priv->read_ref;
+
     switch (obu->header.obu_type) {
     case AV1_OBU_SEQUENCE_HEADER:
         {
@@ -1081,6 +1083,8 @@ static int cbs_av1_write_obu(CodedBitstreamContext *ctx,
     td = NULL;
     start_pos = put_bits_count(pbc);
 
+    priv->ref = (AV1ReferenceFrameState *)&priv->write_ref;
+
     switch (obu->header.obu_type) {
     case AV1_OBU_SEQUENCE_HEADER:
         {
diff --git a/libavcodec/cbs_av1.h b/libavcodec/cbs_av1.h
index c85f58f764..9eaf5c4de6 100644
--- a/libavcodec/cbs_av1.h
+++ b/libavcodec/cbs_av1.h
@@ -441,7 +441,9 @@ typedef struct CodedBitstreamAV1Context {
     int tile_cols;
     int tile_rows;
 
-    AV1ReferenceFrameState ref[AV1_NUM_REF_FRAMES];
+    AV1ReferenceFrameState *ref;
+    AV1ReferenceFrameState read_ref[AV1_NUM_REF_FRAMES];
+    AV1ReferenceFrameState write_ref[AV1_NUM_REF_FRAMES];
 } CodedBitstreamAV1Context;