diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/TimestampAdjuster.java b/library/common/src/main/java/com/google/android/exoplayer2/util/TimestampAdjuster.java
index 8d7b709abdc..4094145f978 100644
--- a/library/common/src/main/java/com/google/android/exoplayer2/util/TimestampAdjuster.java
+++ b/library/common/src/main/java/com/google/android/exoplayer2/util/TimestampAdjuster.java
@@ -105,14 +105,13 @@ public TimestampAdjuster(long firstSampleTimestampUs) {
   public synchronized void sharedInitializeOrWait(boolean canInitialize, long nextSampleTimestampUs)
       throws InterruptedException {
     Assertions.checkState(firstSampleTimestampUs == MODE_SHARED);
-    if (timestampOffsetUs != C.TIME_UNSET) {
-      // Already initialized.
+    if (isInitialized()) {
       return;
     } else if (canInitialize) {
       this.nextSampleTimestampUs.set(nextSampleTimestampUs);
     } else {
       // Wait for another calling thread to complete initialization.
-      while (timestampOffsetUs == C.TIME_UNSET) {
+      while (!isInitialized()) {
         wait();
       }
     }
@@ -194,7 +193,7 @@ public synchronized long adjustSampleTimestamp(long timeUs) {
     if (timeUs == C.TIME_UNSET) {
       return C.TIME_UNSET;
     }
-    if (timestampOffsetUs == C.TIME_UNSET) {
+    if (!isInitialized()) {
       long desiredSampleTimestampUs =
           firstSampleTimestampUs == MODE_SHARED
               ? Assertions.checkNotNull(nextSampleTimestampUs.get())
@@ -207,6 +206,11 @@ public synchronized long adjustSampleTimestamp(long timeUs) {
     return timeUs + timestampOffsetUs;
   }
 
+  /** Returns whether the instance is initialized with a timestamp offset. */
+  public synchronized boolean isInitialized() {
+    return timestampOffsetUs != C.TIME_UNSET;
+  }
+
   /**
    * Converts a 90 kHz clock timestamp to a timestamp in microseconds.
    *
diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java
index 681c8a6d6e0..f646bcadaff 100644
--- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java
+++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java
@@ -681,6 +681,13 @@ private void onEmsgLeafAtomRead(ParsableByteArray atom) {
       pendingMetadataSampleInfos.addLast(
           new MetadataSampleInfo(sampleTimeUs, /* sampleTimeIsRelative= */ false, sampleSize));
       pendingMetadataSampleBytes += sampleSize;
+    } else if (timestampAdjuster != null && !timestampAdjuster.isInitialized()) {
+      // We also need to defer outputting metadata if the timestampAdjuster is not initialized,
+      // else we will set a wrong timestampOffsetUs in timestampAdjuster. See:
+      // https://github.com/androidx/media/issues/356.
+      pendingMetadataSampleInfos.addLast(
+          new MetadataSampleInfo(sampleTimeUs, /* sampleTimeIsRelative= */ false, sampleSize));
+      pendingMetadataSampleBytes += sampleSize;
     } else {
       // We can output the sample metadata immediately.
       if (timestampAdjuster != null) {